From 1e1ba96a3c81377d709af4c79d505ce2ed7df939 Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Thu, 12 Jan 2023 13:46:36 +0100 Subject: [PATCH] Dev/post refactor fixes (#927) * Re-introduce outgoing message logging Signed-off-by: TheArcaneBrony * Websocket dumping * Sentry user count on API * Generate session ID upon opening websocket, fix gateway dumps * Async file io in src/gateway/events/Message.ts Signed-off-by: TheArcaneBrony * Async file io in src/util/util/Config.ts Signed-off-by: TheArcaneBrony * Make pre-commit hook executable Signed-off-by: TheArcaneBrony * Fixed sync file io in src/util/util/Config.ts Signed-off-by: TheArcaneBrony * Fixed missing await call in src/util/util/AutoUpdate.ts Signed-off-by: TheArcaneBrony * Add comment to src/gateway/events/Connection.ts Signed-off-by: TheArcaneBrony * Clean up gateway dumping code Signed-off-by: TheArcaneBrony Co-authored-by: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> --- .gitignore | 1 + .husky/pre-commit | 0 src/api/middlewares/Authentication.ts | 3 +++ src/gateway/events/Connection.ts | 6 +++++- src/gateway/events/Message.ts | 17 +++++++++++++++++ src/gateway/opcodes/Identify.ts | 4 +--- src/gateway/util/Send.ts | 19 +++++++++++++++++++ src/util/util/AutoUpdate.ts | 2 +- src/util/util/Config.ts | 12 ++++++++---- 9 files changed, 55 insertions(+), 9 deletions(-) mode change 100644 => 100755 .husky/pre-commit diff --git a/.gitignore b/.gitignore index 4d1a7a016..9b3520bc6 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ build *.log.ansi *.tmp tmp/ +dump/ \ No newline at end of file diff --git a/.husky/pre-commit b/.husky/pre-commit old mode 100644 new mode 100755 diff --git a/src/api/middlewares/Authentication.ts b/src/api/middlewares/Authentication.ts index ec2c42e54..2aa42e8b5 100644 --- a/src/api/middlewares/Authentication.ts +++ b/src/api/middlewares/Authentication.ts @@ -1,6 +1,7 @@ import { NextFunction, Request, Response } from "express"; import { HTTPError } from "lambert-server"; import { checkToken, Config, Rights } from "@fosscord/util"; +import * as Sentry from "@sentry/node"; export const NO_AUTHORIZATION_ROUTES = [ // Authentication routes @@ -63,6 +64,8 @@ export async function Authentication( if (!req.headers.authorization) return next(new HTTPError("Missing Authorization Header", 401)); + Sentry.setUser({ id: req.user_id }); + try { const { jwtSecret } = Config.get().security; diff --git a/src/gateway/events/Connection.ts b/src/gateway/events/Connection.ts index 41b2ff3db..fff5aacf4 100644 --- a/src/gateway/events/Connection.ts +++ b/src/gateway/events/Connection.ts @@ -1,5 +1,5 @@ import WS from "ws"; -import { WebSocket } from "@fosscord/gateway"; +import { genSessionId, WebSocket } from "@fosscord/gateway"; import { Send } from "../util/Send"; import { CLOSECODES, OPCODES } from "../util/Constants"; import { setHeartbeat } from "../util/Heartbeat"; @@ -30,6 +30,10 @@ export async function Connection( socket.ipAddress = ipAddress; + //Create session ID when the connection is opened. This allows gateway dump to group the initial websocket messages with the rest of the conversation. + const session_id = genSessionId(); + socket.session_id = session_id; //Set the session of the WebSocket object + try { // @ts-ignore socket.on("close", Close); diff --git a/src/gateway/events/Message.ts b/src/gateway/events/Message.ts index 6645dd225..97bbba83a 100644 --- a/src/gateway/events/Message.ts +++ b/src/gateway/events/Message.ts @@ -5,6 +5,8 @@ import WS from "ws"; import { PayloadSchema } from "@fosscord/util"; import * as Sentry from "@sentry/node"; import BigIntJson from "json-bigint"; +import path from "path"; +import fs from "fs/promises"; const bigIntJson = BigIntJson({ storeAsString: true }); var erlpack: any; @@ -41,6 +43,21 @@ export async function Message(this: WebSocket, buffer: WS.Data) { if (process.env.WS_VERBOSE) console.log(`[Websocket] Incomming message: ${JSON.stringify(data)}`); + if (process.env.WS_DUMP) { + const id = this.session_id || "unknown"; + + await fs.mkdir(path.join("dump", this.session_id), { recursive: true }); + await fs.writeFile( + path.join("dump", this.session_id, `${Date.now()}.in.json`), + JSON.stringify(data, null, 2), + ); + + if (!this.session_id) + console.log( + "[Gateway] Unknown session id, dumping to unknown folder", + ); + } + check.call(this, PayloadSchema, data); // @ts-ignore diff --git a/src/gateway/opcodes/Identify.ts b/src/gateway/opcodes/Identify.ts index 29b176436..1c3cab28b 100644 --- a/src/gateway/opcodes/Identify.ts +++ b/src/gateway/opcodes/Identify.ts @@ -55,9 +55,7 @@ export async function onIdentify(this: WebSocket, data: Payload) { return this.close(CLOSECODES.Authentication_failed); } this.user_id = decoded.id; - - const session_id = genSessionId(); - this.session_id = session_id; //Set the session of the WebSocket object + let session_id = this.session_id; const [user, read_states, members, recipients, session, application] = await Promise.all([ diff --git a/src/gateway/util/Send.ts b/src/gateway/util/Send.ts index 1c0f33c3f..c31233c8c 100644 --- a/src/gateway/util/Send.ts +++ b/src/gateway/util/Send.ts @@ -7,8 +7,27 @@ try { ); } import { Payload, WebSocket } from "@fosscord/gateway"; +import fs from "fs/promises"; +import path from "path"; export function Send(socket: WebSocket, data: Payload) { + if (process.env.WS_VERBOSE) + console.log(`[Websocket] Outgoing message: ${JSON.stringify(data)}`); + + if (process.env.WS_DUMP) { + const id = socket.session_id || "unknown"; + + (async () => { + await fs.mkdir(path.join("dump", id), { + recursive: true, + }); + await fs.writeFile( + path.join("dump", id, `${Date.now()}.out.json`), + JSON.stringify(data, null, 2), + ); + })(); + } + let buffer: Buffer | string; if (socket.encoding === "etf") buffer = erlpack.pack(data); // TODO: encode circular object diff --git a/src/util/util/AutoUpdate.ts b/src/util/util/AutoUpdate.ts index 769d959f0..6fbad63ef 100644 --- a/src/util/util/AutoUpdate.ts +++ b/src/util/util/AutoUpdate.ts @@ -60,7 +60,7 @@ async function download(url: string, dir: string) { const response = await fetch(url, { agent }); const buffer = await response.buffer(); const tempDir = await fs.mkdtemp("fosscord"); - fs.writeFile(path.join(tempDir, "Fosscord.zip"), buffer); + await fs.writeFile(path.join(tempDir, "Fosscord.zip"), buffer); } catch (error) { console.error(`[Auto Update] download failed`, error); } diff --git a/src/util/util/Config.ts b/src/util/util/Config.ts index 5d98f5bcc..149f3746c 100644 --- a/src/util/util/Config.ts +++ b/src/util/util/Config.ts @@ -1,5 +1,6 @@ import { ConfigEntity } from "../entities/Config"; -import fs from "fs"; +import fs from "fs/promises"; +import syncFs from "fs"; import { ConfigValue } from "../config"; // TODO: yaml instead of json @@ -31,11 +32,14 @@ export const Config = { ); try { const overrideConfig = JSON.parse( - fs.readFileSync(overridePath, { encoding: "utf8" }), + await fs.readFile(overridePath, { encoding: "utf8" }), ); config = overrideConfig.merge(config); } catch (error) { - fs.writeFileSync(overridePath, JSON.stringify(config, null, 4)); + await fs.writeFile( + overridePath, + JSON.stringify(config, null, 4), + ); } } @@ -79,7 +83,7 @@ function applyConfig(val: ConfigValue) { } if (process.env.CONFIG_PATH) - fs.writeFileSync(overridePath, JSON.stringify(val, null, 4)); + syncFs.writeFileSync(overridePath, JSON.stringify(val, null, 4)); return apply(val); }