From 5ab493105c5acefb919db135e223d68efd2b5ec4 Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Wed, 11 Jan 2023 13:06:35 +0100 Subject: [PATCH 01/11] Re-introduce outgoing message logging Signed-off-by: TheArcaneBrony --- src/gateway/util/Send.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gateway/util/Send.ts b/src/gateway/util/Send.ts index 1c0f33c3f..51236538a 100644 --- a/src/gateway/util/Send.ts +++ b/src/gateway/util/Send.ts @@ -9,6 +9,7 @@ try { import { Payload, WebSocket } from "@fosscord/gateway"; export function Send(socket: WebSocket, data: Payload) { + if (process.env.WS_VERBOSE) console.log(`[Websocket] Outgoing message: ${JSON.stringify(data)}`); let buffer: Buffer | string; if (socket.encoding === "etf") buffer = erlpack.pack(data); // TODO: encode circular object From 8f0d6b325151d0bec8c7b6e4b3c36a733c532e79 Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Wed, 11 Jan 2023 13:21:56 +0100 Subject: [PATCH 02/11] Websocket dumping --- src/gateway/events/Message.ts | 6 ++++++ src/gateway/util/Send.ts | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/src/gateway/events/Message.ts b/src/gateway/events/Message.ts index 6645dd225..8c15e7090 100644 --- a/src/gateway/events/Message.ts +++ b/src/gateway/events/Message.ts @@ -5,6 +5,8 @@ import WS from "ws"; import { PayloadSchema } from "@fosscord/util"; import * as Sentry from "@sentry/node"; import BigIntJson from "json-bigint"; +import path from "path"; +import fs from "fs"; const bigIntJson = BigIntJson({ storeAsString: true }); var erlpack: any; @@ -40,6 +42,10 @@ export async function Message(this: WebSocket, buffer: WS.Data) { if (process.env.WS_VERBOSE) console.log(`[Websocket] Incomming message: ${JSON.stringify(data)}`); + if (process.env.WS_DUMP) { + fs.mkdirSync(path.join("dump", this.session_id), { recursive: true }); + fs.writeFileSync(path.join("dump", this.session_id, `${Date.now()}.in.json`), JSON.stringify(data, null, 2)); + } check.call(this, PayloadSchema, data); diff --git a/src/gateway/util/Send.ts b/src/gateway/util/Send.ts index 51236538a..097468b3e 100644 --- a/src/gateway/util/Send.ts +++ b/src/gateway/util/Send.ts @@ -7,9 +7,15 @@ try { ); } import { Payload, WebSocket } from "@fosscord/gateway"; +import fs from "fs"; +import path from "path"; export function Send(socket: WebSocket, data: Payload) { if (process.env.WS_VERBOSE) console.log(`[Websocket] Outgoing message: ${JSON.stringify(data)}`); + if (process.env.WS_DUMP) { + fs.mkdirSync(path.join("dump", socket.session_id), { recursive: true }); + fs.writeFileSync(path.join("dump", socket.session_id, `${Date.now()}.out.json`), JSON.stringify(data, null, 2)); + } let buffer: Buffer | string; if (socket.encoding === "etf") buffer = erlpack.pack(data); // TODO: encode circular object From d0b93dc0c8e2a54f7913cec7b6357df0cc709a9b Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Wed, 11 Jan 2023 13:22:09 +0100 Subject: [PATCH 03/11] Sentry user count on API --- src/api/middlewares/Authentication.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/api/middlewares/Authentication.ts b/src/api/middlewares/Authentication.ts index ec2c42e54..2aa42e8b5 100644 --- a/src/api/middlewares/Authentication.ts +++ b/src/api/middlewares/Authentication.ts @@ -1,6 +1,7 @@ import { NextFunction, Request, Response } from "express"; import { HTTPError } from "lambert-server"; import { checkToken, Config, Rights } from "@fosscord/util"; +import * as Sentry from "@sentry/node"; export const NO_AUTHORIZATION_ROUTES = [ // Authentication routes @@ -63,6 +64,8 @@ export async function Authentication( if (!req.headers.authorization) return next(new HTTPError("Missing Authorization Header", 401)); + Sentry.setUser({ id: req.user_id }); + try { const { jwtSecret } = Config.get().security; From 69834d4a7fc51176aa6490160818857be529a313 Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Wed, 11 Jan 2023 14:21:03 +0100 Subject: [PATCH 04/11] Generate session ID upon opening websocket, fix gateway dumps --- .gitignore | 1 + src/gateway/events/Connection.ts | 6 +++++- src/gateway/events/Message.ts | 27 +++++++++++++++++---------- src/gateway/opcodes/Identify.ts | 4 +--- src/gateway/util/Send.ts | 12 ++++++++++-- 5 files changed, 34 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 4d1a7a016..9b3520bc6 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ build *.log.ansi *.tmp tmp/ +dump/ \ No newline at end of file diff --git a/src/gateway/events/Connection.ts b/src/gateway/events/Connection.ts index 41b2ff3db..3a386c347 100644 --- a/src/gateway/events/Connection.ts +++ b/src/gateway/events/Connection.ts @@ -1,5 +1,5 @@ import WS from "ws"; -import { WebSocket } from "@fosscord/gateway"; +import { genSessionId, WebSocket } from "@fosscord/gateway"; import { Send } from "../util/Send"; import { CLOSECODES, OPCODES } from "../util/Constants"; import { setHeartbeat } from "../util/Heartbeat"; @@ -30,6 +30,10 @@ export async function Connection( socket.ipAddress = ipAddress; + + const session_id = genSessionId(); + socket.session_id = session_id; //Set the session of the WebSocket object + try { // @ts-ignore socket.on("close", Close); diff --git a/src/gateway/events/Message.ts b/src/gateway/events/Message.ts index 8c15e7090..fc4bea4f4 100644 --- a/src/gateway/events/Message.ts +++ b/src/gateway/events/Message.ts @@ -12,7 +12,7 @@ const bigIntJson = BigIntJson({ storeAsString: true }); var erlpack: any; try { erlpack = require("@yukikaze-bot/erlpack"); -} catch (error) {} +} catch (error) { } export async function Message(this: WebSocket, buffer: WS.Data) { // TODO: compression @@ -43,8 +43,15 @@ export async function Message(this: WebSocket, buffer: WS.Data) { if (process.env.WS_VERBOSE) console.log(`[Websocket] Incomming message: ${JSON.stringify(data)}`); if (process.env.WS_DUMP) { - fs.mkdirSync(path.join("dump", this.session_id), { recursive: true }); - fs.writeFileSync(path.join("dump", this.session_id, `${Date.now()}.in.json`), JSON.stringify(data, null, 2)); + if (this.session_id) { + fs.mkdirSync(path.join("dump", this.session_id), { recursive: true }); + fs.writeFileSync(path.join("dump", this.session_id, `${Date.now()}.in.json`), JSON.stringify(data, null, 2)); + } + else { + fs.mkdirSync(path.join("dump", "unknown"), { recursive: true }); + fs.writeFileSync(path.join("dump", "unknown", `${Date.now()}.in.json`), JSON.stringify(data, null, 2)); + console.log("Unknown session id, dumping to unknown folder"); + } } check.call(this, PayloadSchema, data); @@ -61,13 +68,13 @@ export async function Message(this: WebSocket, buffer: WS.Data) { const transaction = data.op != 1 ? Sentry.startTransaction({ - op: OPCODES[data.op], - name: `GATEWAY ${OPCODES[data.op]}`, - data: { - ...data.d, - token: data?.d?.token ? "[Redacted]" : undefined, - }, - }) + op: OPCODES[data.op], + name: `GATEWAY ${OPCODES[data.op]}`, + data: { + ...data.d, + token: data?.d?.token ? "[Redacted]" : undefined, + }, + }) : undefined; try { diff --git a/src/gateway/opcodes/Identify.ts b/src/gateway/opcodes/Identify.ts index 6244288dd..6c501cf73 100644 --- a/src/gateway/opcodes/Identify.ts +++ b/src/gateway/opcodes/Identify.ts @@ -55,9 +55,7 @@ export async function onIdentify(this: WebSocket, data: Payload) { return this.close(CLOSECODES.Authentication_failed); } this.user_id = decoded.id; - - const session_id = genSessionId(); - this.session_id = session_id; //Set the session of the WebSocket object + let session_id = this.session_id; const [user, read_states, members, recipients, session, application] = await Promise.all([ diff --git a/src/gateway/util/Send.ts b/src/gateway/util/Send.ts index 097468b3e..b45ef6c8e 100644 --- a/src/gateway/util/Send.ts +++ b/src/gateway/util/Send.ts @@ -13,8 +13,16 @@ import path from "path"; export function Send(socket: WebSocket, data: Payload) { if (process.env.WS_VERBOSE) console.log(`[Websocket] Outgoing message: ${JSON.stringify(data)}`); if (process.env.WS_DUMP) { - fs.mkdirSync(path.join("dump", socket.session_id), { recursive: true }); - fs.writeFileSync(path.join("dump", socket.session_id, `${Date.now()}.out.json`), JSON.stringify(data, null, 2)); + if(socket.session_id) { + fs.mkdirSync(path.join("dump", socket.session_id), { recursive: true }); + fs.writeFileSync(path.join("dump", socket.session_id, `${Date.now()}.out.json`), JSON.stringify(data, null, 2)); + } + else { + fs.mkdirSync(path.join("dump", "unknown"), { recursive: true }); + fs.writeFileSync(path.join("dump", "unknown", `${Date.now()}.out.json`), JSON.stringify(data, null, 2)); + console.log("Unknown session ID, dumping to unknown folder!"); + } + } let buffer: Buffer | string; if (socket.encoding === "etf") buffer = erlpack.pack(data); From 2d23f6fed7aed005aad3abbeb9f3e80343725516 Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Wed, 11 Jan 2023 16:28:30 +0100 Subject: [PATCH 05/11] Async file io in src/gateway/events/Message.ts Signed-off-by: TheArcaneBrony --- src/gateway/events/Message.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/gateway/events/Message.ts b/src/gateway/events/Message.ts index fc4bea4f4..197b22009 100644 --- a/src/gateway/events/Message.ts +++ b/src/gateway/events/Message.ts @@ -6,7 +6,7 @@ import { PayloadSchema } from "@fosscord/util"; import * as Sentry from "@sentry/node"; import BigIntJson from "json-bigint"; import path from "path"; -import fs from "fs"; +import fs from "fs/promises"; const bigIntJson = BigIntJson({ storeAsString: true }); var erlpack: any; @@ -44,12 +44,12 @@ export async function Message(this: WebSocket, buffer: WS.Data) { console.log(`[Websocket] Incomming message: ${JSON.stringify(data)}`); if (process.env.WS_DUMP) { if (this.session_id) { - fs.mkdirSync(path.join("dump", this.session_id), { recursive: true }); - fs.writeFileSync(path.join("dump", this.session_id, `${Date.now()}.in.json`), JSON.stringify(data, null, 2)); + await fs.mkdir(path.join("dump", this.session_id), { recursive: true }); + await fs.writeFile(path.join("dump", this.session_id, `${Date.now()}.in.json`), JSON.stringify(data, null, 2)); } else { - fs.mkdirSync(path.join("dump", "unknown"), { recursive: true }); - fs.writeFileSync(path.join("dump", "unknown", `${Date.now()}.in.json`), JSON.stringify(data, null, 2)); + await fs.mkdir(path.join("dump", "unknown"), { recursive: true }); + await fs.writeFile(path.join("dump", "unknown", `${Date.now()}.in.json`), JSON.stringify(data, null, 2)); console.log("Unknown session id, dumping to unknown folder"); } } From ef3ff801fc09afd8a27bd2ab9b947af277734a2f Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Wed, 11 Jan 2023 16:30:46 +0100 Subject: [PATCH 06/11] Async file io in src/util/util/Config.ts Signed-off-by: TheArcaneBrony --- src/util/util/Config.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/util/util/Config.ts b/src/util/util/Config.ts index 5d98f5bcc..2315b458a 100644 --- a/src/util/util/Config.ts +++ b/src/util/util/Config.ts @@ -1,5 +1,5 @@ import { ConfigEntity } from "../entities/Config"; -import fs from "fs"; +import fs from "fs/promises"; import { ConfigValue } from "../config"; // TODO: yaml instead of json @@ -31,11 +31,11 @@ export const Config = { ); try { const overrideConfig = JSON.parse( - fs.readFileSync(overridePath, { encoding: "utf8" }), + await fs.readFile(overridePath, { encoding: "utf8" }), ); config = overrideConfig.merge(config); } catch (error) { - fs.writeFileSync(overridePath, JSON.stringify(config, null, 4)); + await fs.writeFile(overridePath, JSON.stringify(config, null, 4)); } } From 9e0893686132ba5560c61a61811b1e6816ff294c Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Wed, 11 Jan 2023 16:31:35 +0100 Subject: [PATCH 07/11] Make pre-commit hook executable Signed-off-by: TheArcaneBrony --- .husky/pre-commit | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .husky/pre-commit diff --git a/.husky/pre-commit b/.husky/pre-commit old mode 100644 new mode 100755 From 8728b756e04157fe85279624d6c47cbaa70370c0 Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Wed, 11 Jan 2023 16:32:52 +0100 Subject: [PATCH 08/11] Fixed sync file io in src/util/util/Config.ts Signed-off-by: TheArcaneBrony --- src/util/util/Config.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/util/util/Config.ts b/src/util/util/Config.ts index 2315b458a..149f3746c 100644 --- a/src/util/util/Config.ts +++ b/src/util/util/Config.ts @@ -1,5 +1,6 @@ import { ConfigEntity } from "../entities/Config"; import fs from "fs/promises"; +import syncFs from "fs"; import { ConfigValue } from "../config"; // TODO: yaml instead of json @@ -35,7 +36,10 @@ export const Config = { ); config = overrideConfig.merge(config); } catch (error) { - await fs.writeFile(overridePath, JSON.stringify(config, null, 4)); + await fs.writeFile( + overridePath, + JSON.stringify(config, null, 4), + ); } } @@ -79,7 +83,7 @@ function applyConfig(val: ConfigValue) { } if (process.env.CONFIG_PATH) - fs.writeFileSync(overridePath, JSON.stringify(val, null, 4)); + syncFs.writeFileSync(overridePath, JSON.stringify(val, null, 4)); return apply(val); } From 0d69d8d1481e9d8b4745918d93b23ff9a56e3eae Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Wed, 11 Jan 2023 16:34:04 +0100 Subject: [PATCH 09/11] Fixed missing await call in src/util/util/AutoUpdate.ts Signed-off-by: TheArcaneBrony --- src/util/util/AutoUpdate.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/util/AutoUpdate.ts b/src/util/util/AutoUpdate.ts index 769d959f0..6fbad63ef 100644 --- a/src/util/util/AutoUpdate.ts +++ b/src/util/util/AutoUpdate.ts @@ -60,7 +60,7 @@ async function download(url: string, dir: string) { const response = await fetch(url, { agent }); const buffer = await response.buffer(); const tempDir = await fs.mkdtemp("fosscord"); - fs.writeFile(path.join(tempDir, "Fosscord.zip"), buffer); + await fs.writeFile(path.join(tempDir, "Fosscord.zip"), buffer); } catch (error) { console.error(`[Auto Update] download failed`, error); } From a5d7dca4fe84d9e67062b162eed2ea17145301a6 Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Wed, 11 Jan 2023 16:36:21 +0100 Subject: [PATCH 10/11] Add comment to src/gateway/events/Connection.ts Signed-off-by: TheArcaneBrony --- src/gateway/events/Connection.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gateway/events/Connection.ts b/src/gateway/events/Connection.ts index 3a386c347..fff5aacf4 100644 --- a/src/gateway/events/Connection.ts +++ b/src/gateway/events/Connection.ts @@ -30,10 +30,10 @@ export async function Connection( socket.ipAddress = ipAddress; - + //Create session ID when the connection is opened. This allows gateway dump to group the initial websocket messages with the rest of the conversation. const session_id = genSessionId(); socket.session_id = session_id; //Set the session of the WebSocket object - + try { // @ts-ignore socket.on("close", Close); From cb59a977fdd2d0004d75b6d6e913c610abedcd68 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Thu, 12 Jan 2023 13:50:38 +1100 Subject: [PATCH 11/11] Clean up gateway dumping code --- src/gateway/events/Message.ts | 38 +++++++++++++++++++---------------- src/gateway/util/Send.ts | 28 +++++++++++++++----------- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/src/gateway/events/Message.ts b/src/gateway/events/Message.ts index 197b22009..97bbba83a 100644 --- a/src/gateway/events/Message.ts +++ b/src/gateway/events/Message.ts @@ -12,7 +12,7 @@ const bigIntJson = BigIntJson({ storeAsString: true }); var erlpack: any; try { erlpack = require("@yukikaze-bot/erlpack"); -} catch (error) { } +} catch (error) {} export async function Message(this: WebSocket, buffer: WS.Data) { // TODO: compression @@ -42,16 +42,20 @@ export async function Message(this: WebSocket, buffer: WS.Data) { if (process.env.WS_VERBOSE) console.log(`[Websocket] Incomming message: ${JSON.stringify(data)}`); + if (process.env.WS_DUMP) { - if (this.session_id) { - await fs.mkdir(path.join("dump", this.session_id), { recursive: true }); - await fs.writeFile(path.join("dump", this.session_id, `${Date.now()}.in.json`), JSON.stringify(data, null, 2)); - } - else { - await fs.mkdir(path.join("dump", "unknown"), { recursive: true }); - await fs.writeFile(path.join("dump", "unknown", `${Date.now()}.in.json`), JSON.stringify(data, null, 2)); - console.log("Unknown session id, dumping to unknown folder"); - } + const id = this.session_id || "unknown"; + + await fs.mkdir(path.join("dump", this.session_id), { recursive: true }); + await fs.writeFile( + path.join("dump", this.session_id, `${Date.now()}.in.json`), + JSON.stringify(data, null, 2), + ); + + if (!this.session_id) + console.log( + "[Gateway] Unknown session id, dumping to unknown folder", + ); } check.call(this, PayloadSchema, data); @@ -68,13 +72,13 @@ export async function Message(this: WebSocket, buffer: WS.Data) { const transaction = data.op != 1 ? Sentry.startTransaction({ - op: OPCODES[data.op], - name: `GATEWAY ${OPCODES[data.op]}`, - data: { - ...data.d, - token: data?.d?.token ? "[Redacted]" : undefined, - }, - }) + op: OPCODES[data.op], + name: `GATEWAY ${OPCODES[data.op]}`, + data: { + ...data.d, + token: data?.d?.token ? "[Redacted]" : undefined, + }, + }) : undefined; try { diff --git a/src/gateway/util/Send.ts b/src/gateway/util/Send.ts index b45ef6c8e..c31233c8c 100644 --- a/src/gateway/util/Send.ts +++ b/src/gateway/util/Send.ts @@ -7,23 +7,27 @@ try { ); } import { Payload, WebSocket } from "@fosscord/gateway"; -import fs from "fs"; +import fs from "fs/promises"; import path from "path"; export function Send(socket: WebSocket, data: Payload) { - if (process.env.WS_VERBOSE) console.log(`[Websocket] Outgoing message: ${JSON.stringify(data)}`); + if (process.env.WS_VERBOSE) + console.log(`[Websocket] Outgoing message: ${JSON.stringify(data)}`); + if (process.env.WS_DUMP) { - if(socket.session_id) { - fs.mkdirSync(path.join("dump", socket.session_id), { recursive: true }); - fs.writeFileSync(path.join("dump", socket.session_id, `${Date.now()}.out.json`), JSON.stringify(data, null, 2)); - } - else { - fs.mkdirSync(path.join("dump", "unknown"), { recursive: true }); - fs.writeFileSync(path.join("dump", "unknown", `${Date.now()}.out.json`), JSON.stringify(data, null, 2)); - console.log("Unknown session ID, dumping to unknown folder!"); - } - + const id = socket.session_id || "unknown"; + + (async () => { + await fs.mkdir(path.join("dump", id), { + recursive: true, + }); + await fs.writeFile( + path.join("dump", id, `${Date.now()}.out.json`), + JSON.stringify(data, null, 2), + ); + })(); } + let buffer: Buffer | string; if (socket.encoding === "etf") buffer = erlpack.pack(data); // TODO: encode circular object