Add test for joining a new federated room (#31)

Add test for joining a new federated room and making sure the messages are available (homeserver should backfill).

Synapse changes: matrix-org/synapse#13205, matrix-org/synapse#13320

server/ensure-room-joined.js

@@ -0,0 +1,32 @@
+'use strict';
+
+const assert = require('assert');
+const urlJoin = require('url-join');
+
+const { fetchEndpointAsJson } = require('./lib/fetch-endpoint');
+
+const config = require('./lib/config');
+const matrixServerUrl = config.get('matrixServerUrl');
+assert(matrixServerUrl);
+
+async function ensureRoomJoined(accessToken, roomId, viaServers = []) {
+  let qs = new URLSearchParams();
+  [].concat(viaServers).forEach((viaServer) => {
+    qs.append('server_name', viaServer);
+  });
+
+  // TODO: Only join world_readable rooms. Perhaps we want to serve public rooms
+  // where we have been invited. GET
+  // /_matrix/client/v3/directory/list/room/{roomId} (Gets the visibility of a
+  // given room on the server’s public room directory.)
+  const joinEndpoint = urlJoin(
+    matrixServerUrl,
+    `_matrix/client/r0/join/${roomId}?${qs.toString()}`
+  );
+  await fetchEndpointAsJson(joinEndpoint, {
+    method: 'POST',
+    accessToken,
+  });
+}
+
+module.exports = ensureRoomJoined;

server/fetch-events-in-range.js

@@ -27,16 +27,6 @@ async function fetchEventsFromTimestampBackwards(accessToken, roomId, ts, limit)
   assert(ts);
   assert(limit);
 
-  // TODO: Only join world_readable rooms. Perhaps we want to serve public rooms
-  // where we have been invited. GET
-  // /_matrix/client/v3/directory/list/room/{roomId} (Gets the visibility of a
-  // given room on the server’s public room directory.)
-  const joinEndpoint = urlJoin(matrixServerUrl, `_matrix/client/r0/join/${roomId}`);
-  await fetchEndpointAsJson(joinEndpoint, {
-    method: 'POST',
-    accessToken,
-  });
-
   const timestampToEventEndpoint = urlJoin(
     matrixServerUrl,
     `_matrix/client/unstable/org.matrix.msc3030/rooms/${roomId}/timestamp_to_event?ts=${ts}&dir=b`

server/routes/install-routes.js

@@ -12,6 +12,7 @@ const timeoutMiddleware = require('./timeout-middleware');
 
 const fetchRoomData = require('../fetch-room-data');
 const fetchEventsInRange = require('../fetch-events-in-range');
+const ensureRoomJoined = require('../ensure-room-joined');
 const renderHydrogenToString = require('../hydrogen-render/1-render-hydrogen-to-string');
 const sanitizeHtml = require('../lib/sanitize-html');
 const safeJson = require('../lib/safe-json');
@@ -136,23 +137,34 @@ function installRoutes(app) {
       // If the hourRange is defined, we force the range to always be 1 hour. If
       // the format isn't correct, redirect to the correct hour range
       if (hourRange && toHour !== fromHour + 1) {
+        // Pass through the query parameters
+        let queryParamterUrlPiece = '';
+        if (req.query) {
+          queryParamterUrlPiece = `?${new URLSearchParams(req.query).toString()}`;
+        }
+
         res.redirect(
-          urlJoin(
+          // FIXME: Can we use the matrixPublicArchiveURLCreator here?
+          `${urlJoin(
             basePath,
             roomIdOrAlias,
             'date',
             req.params.yyyy,
             req.params.mm,
             req.params.dd,
             `${fromHour}-${fromHour + 1}`
-          )
+          )}${queryParamterUrlPiece}`
         );
         return;
       }
 
       // TODO: Highlight tile that matches ?at=$xxx
       //const aroundId = req.query.at;
 
+      // We have to wait for the room join to happen first before we can fetch
+      // any of the additional room info or messages.
+      await ensureRoomJoined(matrixAccessToken, roomIdOrAlias, req.query.via);
+
       // Do these in parallel to avoid the extra time in sequential round-trips
       // (we want to display the archive page faster)
       const [roomData, { events, stateEventMap }] = await Promise.all([

shared/lib/url-creator.js

@@ -7,12 +7,17 @@ class URLCreator {
     this._basePath = basePath;
   }
 
-  archiveUrlForDate(roomId, date) {
+  archiveUrlForDate(roomId, date, { viaServers = [] } = {}) {
+    let qs = new URLSearchParams();
+    [].concat(viaServers).forEach((viaServer) => {
+      qs.append('via', viaServer);
+    });
+
     // Gives the date in YYYY/mm/dd format.
     // date.toISOString() -> 2022-02-16T23:20:04.709Z
     const urlDate = date.toISOString().split('T')[0].replaceAll('-', '/');
 
-    return urlJoin(this._basePath, `${roomId}/date/${urlDate}`);
+    return `${urlJoin(this._basePath, `${roomId}/date/${urlDate}`)}?${qs.toString()}`;
   }
 }
 

test/client-utils.js

@@ -1,20 +1,45 @@
 'use strict';
 
 const assert = require('assert');
-const { URLSearchParams } = require('url');
 const urlJoin = require('url-join');
 const { fetchEndpointAsJson, fetchEndpoint } = require('../server/lib/fetch-endpoint');
 
 const config = require('../server/lib/config');
 const matrixAccessToken = config.get('matrixAccessToken');
 assert(matrixAccessToken);
+const testMatrixServerUrl1 = config.get('testMatrixServerUrl1');
+assert(testMatrixServerUrl1);
 
 let txnCount = 0;
 function getTxnId() {
   txnCount++;
   return `${new Date().getTime()}--${txnCount}`;
 }
 
+async function ensureUserRegistered({ matrixServerUrl, username }) {
+  const registerResponse = await fetchEndpointAsJson(
+    urlJoin(matrixServerUrl, '/_matrix/client/v3/register'),
+    {
+      method: 'POST',
+      body: {
+        type: 'm.login.dummy',
+        username,
+      },
+    }
+  );
+
+  const userId = registerResponse['user_id'];
+  assert(userId);
+}
+
+async function getTestClientForAs() {
+  return {
+    homeserverUrl: testMatrixServerUrl1,
+    accessToken: matrixAccessToken,
+    userId: '@archiver:hs1',
+  };
+}
+
 // Get client to act with for all of the client methods. This will use the
 // application service access token and client methods will append `?user_id`
 // for the specific user to act upon so we can use the `?ts` message timestamp
@@ -92,13 +117,15 @@ async function joinRoom({ client, roomId, viaServers }) {
     qs.append('user_id', client.applicationServiceUserIdOverride);
   }
 
-  const joinRoomResponse = await fetchEndpointAsJson(
-    urlJoin(client.homeserverUrl, `/_matrix/client/v3/join/${roomId}?${qs.toString()}`),
-    {
-      method: 'POST',
-      accessToken: client.accessToken,
-    }
+  const joinRoomUrl = urlJoin(
+    client.homeserverUrl,
+    `/_matrix/client/v3/join/${roomId}?${qs.toString()}`
   );
+  console.log('test client joinRoomUrl', joinRoomUrl);
+  const joinRoomResponse = await fetchEndpointAsJson(joinRoomUrl, {
+    method: 'POST',
+    accessToken: client.accessToken,
+  });
 
   const joinedRoomId = joinRoomResponse['room_id'];
   assert(joinedRoomId);
@@ -164,11 +191,19 @@ async function createMessagesInRoom({ client, roomId, numMessages, prefix, times
         msgtype: 'm.text',
         body: `${prefix} - message${i}`,
       },
-      timestamp,
+      // We can't use the exact same timestamp for every message in the tests
+      // otherwise it's a toss up which event will be returned as the closest
+      // for `/timestamp_to_event`. As a note, we don't have to do this after
+      // https://github.com/matrix-org/synapse/pull/13658 merges but it still
+      // seems like a good idea to make the tests more clear.
+      timestamp: timestamp + i,
     });
     eventIds.push(eventId);
   }
 
+  // Sanity check that we actually sent some messages
+  assert.strictEqual(eventIds.length, numMessages);
+
   return eventIds;
 }
 
@@ -178,33 +213,39 @@ async function updateProfile({ client, displayName, avatarUrl }) {
     qs.append('user_id', client.applicationServiceUserIdOverride);
   }
 
-  const updateDisplayNamePromise = fetchEndpointAsJson(
-    urlJoin(
-      client.homeserverUrl,
-      `/_matrix/client/v3/profile/${client.userId}/displayname?${qs.toString()}`
-    ),
-    {
-      method: 'PUT',
-      body: {
-        displayname: displayName,
-      },
-      accessToken: client.accessToken,
-    }
-  );
+  let updateDisplayNamePromise = Promise.resolve();
+  if (displayName) {
+    updateDisplayNamePromise = fetchEndpointAsJson(
+      urlJoin(
+        client.homeserverUrl,
+        `/_matrix/client/v3/profile/${client.userId}/displayname?${qs.toString()}`
+      ),
+      {
+        method: 'PUT',
+        body: {
+          displayname: displayName,
+        },
+        accessToken: client.accessToken,
+      }
+    );
+  }
 
-  const updateAvatarUrlPromise = fetchEndpointAsJson(
-    urlJoin(
-      client.homeserverUrl,
-      `/_matrix/client/v3/profile/${client.userId}/avatar_url?${qs.toString()}`
-    ),
-    {
-      method: 'PUT',
-      body: {
-        avatar_url: avatarUrl,
-      },
-      accessToken: client.accessToken,
-    }
-  );
+  let updateAvatarUrlPromise = Promise.resolve();
+  if (avatarUrl) {
+    updateAvatarUrlPromise = fetchEndpointAsJson(
+      urlJoin(
+        client.homeserverUrl,
+        `/_matrix/client/v3/profile/${client.userId}/avatar_url?${qs.toString()}`
+      ),
+      {
+        method: 'PUT',
+        body: {
+          avatar_url: avatarUrl,
+        },
+        accessToken: client.accessToken,
+      }
+    );
+  }
 
   await Promise.all([updateDisplayNamePromise, updateAvatarUrlPromise]);
 
@@ -248,6 +289,8 @@ async function uploadContent({ client, roomId, data, fileName, contentType }) {
 }
 
 module.exports = {
+  ensureUserRegistered,
+  getTestClientForAs,
   getTestClientForHs,
   createTestRoom,
   joinRoom,

test/e2e-tests.js

@@ -14,6 +14,7 @@ const { fetchEndpointAsText, fetchEndpointAsJson } = require('../server/lib/fetc
 const config = require('../server/lib/config');
 
 const {
+  getTestClientForAs,
   getTestClientForHs,
   createTestRoom,
   joinRoom,
@@ -106,6 +107,20 @@ describe('matrix-public-archive', () => {
   });
 
   describe('Archive', () => {
+    before(async () => {
+      // Make sure the application service archiver user itself has a profile
+      // set otherwise we run into 404, `Profile was not found` errors when
+      // joining a remote federated room from the archiver user, see
+      // https://github.com/matrix-org/synapse/issues/4778
+      //
+      // FIXME: Remove after https://github.com/matrix-org/synapse/issues/4778 is resolved
+      const asClient = await getTestClientForAs();
+      await updateProfile({
+        client: asClient,
+        displayName: 'Archiver',
+      });
+    });
+
     // Use a fixed date at the start of the UTC day so that the tests are
     // consistent. Otherwise, the tests could fail when they start close to
     // midnight and it rolls over to the next day.
@@ -163,6 +178,7 @@ describe('matrix-public-archive', () => {
         `Coulomb's Law of Friction: Kinetic friction is independent of the sliding velocity.`,
       ];
 
+      // TODO: Can we use `createMessagesInRoom` here instead?
       const eventIds = [];
       for (const messageText of messageTextList) {
         const eventId = await sendMessageOnArchiveDate({
@@ -375,7 +391,39 @@ describe('matrix-public-archive', () => {
       );
     });
 
-    it(`can render day back in time from room on remote homeserver we haven't backfilled from`);
+    it(`can render day back in time from room on remote homeserver we haven't backfilled from`, async () => {
+      const hs2Client = await getTestClientForHs(testMatrixServerUrl2);
+
+      // Create a room on hs2
+      const hs2RoomId = await createTestRoom(hs2Client);
+      const room2EventIds = await createMessagesInRoom({
+        client: hs2Client,
+        roomId: hs2RoomId,
+        numMessages: 3,
+        prefix: HOMESERVER_URL_TO_PRETTY_NAME_MAP[hs2Client.homeserverUrl],
+        timestamp: archiveDate.getTime(),
+      });
+
+      archiveUrl = matrixPublicArchiveURLCreator.archiveUrlForDate(hs2RoomId, archiveDate, {
+        // Since hs1 doesn't know about this room on hs2 yet, we have to provide
+        // a via server to ask through.
+        viaServers: ['hs2'],
+      });
+
+      const archivePageHtml = await fetchEndpointAsText(archiveUrl);
+
+      const dom = parseHTML(archivePageHtml);
+
+      // Make sure the messages are visible
+      assert.deepStrictEqual(
+        room2EventIds.map((eventId) => {
+          return dom.document
+            .querySelector(`[data-event-id="${eventId}"]`)
+            ?.getAttribute('data-event-id');
+        }),
+        room2EventIds
+      );
+    });
 
     it(`will redirect to hour pagination when there are too many messages`);