chore: move to matrix-* (#33525)

Co-authored-by: Debdut Chakraborty <debdutdeb@outlook.com>
Co-authored-by: Debdut Chakraborty <debdut.chakraborty@rocket.chat>

.github/actions/build-docker/action.yml

@@ -85,6 +85,12 @@ runs:
       if: inputs.setup == 'true'
       shell: bash
 
+    - if: ${{ matrix.platform == 'alpine' }}
+      uses: actions/download-artifact@v4
+      with:
+        name: napi-binary
+        path: /tmp/build/matrix-sdk-crypto.linux-x64-musl.node
+
     - name: Build Docker images
       shell: bash
       run: |

.github/workflows/ci.yml

@@ -280,9 +280,46 @@ jobs:
           deno-version: ${{ needs.release-versions.outputs.deno-version }}
           coverage: ${{ github.event_name != 'release' }}
 
+  # TODO: this should go away once upstream builds are fixed
+  build-matrix-rust-bindings-for-alpine:
+    name: Builds matrix rust bindings against alpine
+    runs-on: ubuntu-latest
+    steps:
+      - run: sudo apt-get install -y musl-tools libunwind-dev && find /usr/include -name stdarg.h 2>/dev/null || true
+      - uses: actions/checkout@v4
+        with:
+          repository: matrix-org/matrix-rust-sdk-crypto-nodejs
+          ref: v0.2.0-beta.1 # https://github.com/element-hq/matrix-bot-sdk/blob/e72a4c498e00c6c339a791630c45d00a351f56a8/package.json#L58
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: stable
+          targets: x86_64-unknown-linux-musl
+
+      - name: Install ziglang
+        uses: mlugg/setup-zig@v1
+        with:
+          version: 0.13.0
+
+      - name: Build
+        run: |
+          npm install --ignore-scripts
+          npx napi build --release --target x86_64-unknown-linux-musl --platform --zig
+
+      - name: Upload bin
+        uses: actions/upload-artifact@v4
+        with:
+          path: matrix-sdk-crypto.linux-x64-musl.node
+          name: napi-binary
+          if-no-files-found: error
+
   build-gh-docker-coverage:
     name: 🚢 Build Docker Images for Testing
-    needs: [build, release-versions]
+    needs: [build, release-versions, build-matrix-rust-bindings-for-alpine]
     runs-on: ubuntu-20.04
 
     env:
@@ -311,6 +348,11 @@ jobs:
           build-containers: ${{ matrix.platform == 'alpine' && 'authorization-service account-service ddp-streamer-service presence-service stream-hub-service queue-worker-service omnichannel-transcript-service' || '' }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
+      - name: Make sure matrix bindings load
+        if: ${{ matrix.platform == 'alpine' }}
+        run: |
+          docker run --rm -w /app/bundle/programs/server/npm/node_modules/matrix-appservice-bridge ghcr.io/rocketchat/rocket.chat:$RC_DOCKER_TAG -e 'require(".")'
+
   build-gh-docker:
     name: 🚢 Build Docker Images for Production
     needs: [build-prod, release-versions]

apps/meteor/.docker/Dockerfile.alpine

@@ -36,6 +36,10 @@ RUN set -x \
     && npm cache clear --force \
     && apk del .fetch-deps
 
+# TODO: remove hack once upstream builds are fixed
+COPY matrix-sdk-crypto.linux-x64-musl.node /app/bundle/programs/server/npm/node_modules/@matrix-org/matrix-sdk-crypto-nodejs
+COPY matrix-sdk-crypto.linux-x64-musl.node /app/bundle/programs/server/npm/node_modules/@vector-im/matrix-bot-sdk/node_modules/@matrix-org/matrix-sdk-crypto-nodejs
+
 VOLUME /app/uploads
 
 WORKDIR /app/bundle

apps/meteor/package.json

@@ -241,8 +241,6 @@
 		"@rocket.chat/css-in-js": "~0.31.25",
 		"@rocket.chat/emitter": "~0.31.25",
 		"@rocket.chat/favicon": "workspace:^",
-		"@rocket.chat/forked-matrix-appservice-bridge": "^4.0.2",
-		"@rocket.chat/forked-matrix-bot-sdk": "^0.6.0-beta.3",
 		"@rocket.chat/freeswitch": "workspace:^",
 		"@rocket.chat/fuselage": "^0.59.3",
 		"@rocket.chat/fuselage-hooks": "^0.33.1",
@@ -292,6 +290,7 @@
 		"@slack/bolt": "^3.14.0",
 		"@slack/rtm-api": "^6.0.0",
 		"@tanstack/react-query": "^4.16.1",
+		"@vector-im/matrix-bot-sdk": "0.7.1-element.6",
 		"@xmldom/xmldom": "^0.8.10",
 		"adm-zip": "0.5.16",
 		"ajv": "^8.11.0",
@@ -370,6 +369,8 @@
 		"lodash.get": "^4.4.2",
 		"mailparser": "^3.4.0",
 		"marked": "^4.2.5",
+		"matrix-appservice": "^2.0.0",
+		"matrix-appservice-bridge": "^10.3.1",
 		"mem": "^8.1.1",
 		"meteor-node-stubs": "^1.2.10",
 		"mime-db": "^1.52.0",

apps/meteor/server/services/federation/application/AbstractFederationApplicationService.ts

@@ -54,7 +54,10 @@ export abstract class AbstractFederationApplicationService {
 			return;
 		}
 		if (federatedUser.shouldUpdateFederationAvatar(avatarUrl)) {
-			await this.internalUserAdapter.setAvatar(federatedUser, this.bridge.convertMatrixUrlToHttp(federatedUser.getExternalId(), avatarUrl));
+			await this.internalUserAdapter.setAvatar(
+				federatedUser,
+				await this.bridge.convertMatrixUrlToHttp(federatedUser.getExternalId(), avatarUrl),
+			);
 			await this.internalUserAdapter.updateFederationAvatar(federatedUser.getInternalId(), avatarUrl);
 		}
 	}

apps/meteor/server/services/federation/domain/IFederationBridge.ts

@@ -82,7 +82,7 @@ export interface IFederationBridge {
 		fileDetails: { filename: string; fileSize: number; mimeType: string; metadata?: { width?: number; height?: number; format?: string } },
 	): Promise<string>;
 	uploadContent(externalSenderId: string, content: Buffer, options?: { name?: string; type?: string }): Promise<string | undefined>;
-	convertMatrixUrlToHttp(externalUserId: string, matrixUrl: string): string;
+	convertMatrixUrlToHttp(externalUserId: string, matrixUrl: string): Promise<string>;
 	sendReplyToMessage(
 		externalRoomId: string,
 		externalUserId: string,

apps/meteor/server/services/federation/infrastructure/matrix/Bridge.ts

@@ -1,6 +1,6 @@
 import type { IMessage } from '@rocket.chat/core-typings';
-import type { AppServiceOutput, Bridge } from '@rocket.chat/forked-matrix-appservice-bridge';
 import { serverFetch as fetch } from '@rocket.chat/server-fetch';
+import type { AppServiceOutput, Bridge } from 'matrix-appservice-bridge';
 
 import type { IExternalUserProfileInformation, IFederationBridge, IFederationBridgeRegistrationFile } from '../../domain/IFederationBridge';
 import type { RocketChatSettingsAdapter } from '../rocket-chat/adapters/Settings';
@@ -50,7 +50,7 @@ export class MatrixBridge implements IFederationBridge {
 				this.bridgeInstance.addAppServicePath({
 					method: 'POST',
 					path: '/_matrix/app/v1/ping',
-					checkToken: true,
+					authenticate: true,
 					handler: (_req, res, _next) => {
 						/*
 						 * https://spec.matrix.org/v1.11/application-service-api/#post_matrixappv1ping
@@ -489,7 +489,7 @@ export class MatrixBridge implements IFederationBridge {
 	}
 
 	public async getReadStreamForFileFromUrl(externalUserId: string, fileUrl: string): Promise<ReadableStream> {
-		const response = await fetch(this.convertMatrixUrlToHttp(externalUserId, fileUrl));
+		const response = await fetch(await this.convertMatrixUrlToHttp(externalUserId, fileUrl));
 		if (!response.body) {
 			throw new Error('Not able to download the file');
 		}
@@ -736,15 +736,15 @@ export class MatrixBridge implements IFederationBridge {
 		await this.bridgeInstance.getIntent(externalUserId).setRoomTopic(externalRoomId, roomTopic);
 	}
 
-	public convertMatrixUrlToHttp(externalUserId: string, matrixUrl: string): string {
+	public convertMatrixUrlToHttp(externalUserId: string, matrixUrl: string): Promise<string> {
 		return this.bridgeInstance.getIntent(externalUserId).matrixClient.mxcToHttp(matrixUrl);
 	}
 
 	protected async createInstance(): Promise<void> {
 		federationBridgeLogger.info('Performing Dynamic Import of matrix-appservice-bridge');
 
 		// Dynamic import to prevent Rocket.Chat from loading the module until needed and then handle if that fails
-		const { Bridge, AppServiceRegistration, MatrixUser } = await import('@rocket.chat/forked-matrix-appservice-bridge');
+		const { Bridge, AppServiceRegistration, MatrixUser } = await import('matrix-appservice-bridge');
 		MatrixUserInstance = MatrixUser;
 		const registrationFile = this.internalSettings.getAppServiceRegistrationObject();
 

apps/meteor/server/services/federation/infrastructure/matrix/converters/room/to-internal-parser-formatter.ts

@@ -1,4 +1,4 @@
-import type { MentionPill as MentionPillType } from '@rocket.chat/forked-matrix-bot-sdk';
+import type { MentionPill as MentionPillType } from '@vector-im/matrix-bot-sdk';
 import { marked } from 'marked';
 
 const INTERNAL_MENTIONS_FOR_EXTERNAL_USERS_REGEX = /@([0-9a-zA-Z-_.]+(@([0-9a-zA-Z-_.]+))?):+([0-9a-zA-Z-_.]+)(?=[^<>]*(?:<\w|$))/gm; // @username:server.com excluding any <a> tags
@@ -20,23 +20,23 @@ const replaceMessageMentions = async (
 };
 
 const replaceMentionsFromLocalExternalUsersForExternalFormat = async (message: string): Promise<string> => {
-	const { MentionPill } = await import('@rocket.chat/forked-matrix-bot-sdk');
+	const { MentionPill } = await import('@vector-im/matrix-bot-sdk');
 
 	return replaceMessageMentions(message, INTERNAL_MENTIONS_FOR_EXTERNAL_USERS_REGEX, (match: string) =>
 		MentionPill.forUser(match.trimStart()),
 	);
 };
 
 const replaceInternalUsersMentionsForExternalFormat = async (message: string, homeServerDomain: string): Promise<string> => {
-	const { MentionPill } = await import('@rocket.chat/forked-matrix-bot-sdk');
+	const { MentionPill } = await import('@vector-im/matrix-bot-sdk');
 
 	return replaceMessageMentions(message, INTERNAL_MENTIONS_FOR_INTERNAL_USERS_REGEX, (match: string) =>
 		MentionPill.forUser(`${match.trimStart()}:${homeServerDomain}`),
 	);
 };
 
 const replaceInternalGeneralMentionsForExternalFormat = async (message: string, externalRoomId: string): Promise<string> => {
-	const { MentionPill } = await import('@rocket.chat/forked-matrix-bot-sdk');
+	const { MentionPill } = await import('@vector-im/matrix-bot-sdk');
 
 	return replaceMessageMentions(message, INTERNAL_GENERAL_REGEX, () => MentionPill.forRoom(externalRoomId));
 };
@@ -79,7 +79,7 @@ export const toExternalQuoteMessageFormat = async ({
 	message: string;
 	homeServerDomain: string;
 }): Promise<{ message: string; formattedMessage: string }> => {
-	const { RichReply } = await import('@rocket.chat/forked-matrix-bot-sdk');
+	const { RichReply } = await import('@vector-im/matrix-bot-sdk');
 
 	const formattedMessage = convertMarkdownToHTML(message);
 	const finalFormattedMessage = convertMarkdownToHTML(

apps/meteor/tests/data/permissions.helper.ts

@@ -30,15 +30,26 @@ const updateManyPermissions = (permissions: { [key: string]: string[] }): Promis
 			.end((err?: Error) => setTimeout(() => (!err && resolve()) || reject(err), 100));
 	});
 
-export const updateSetting = (setting: string, value: ISetting['value']): Promise<void | Error> =>
+export const updateSetting = (setting: string, value: ISetting['value'], debounce = true): Promise<void | Error> =>
 	new Promise((resolve, reject) => {
 		void request
 			.post(`/api/v1/settings/${setting}`)
 			.set(credentials)
 			.send({ value })
 			.expect('Content-Type', 'application/json')
 			.expect(200)
-			.end((err?: Error) => setTimeout(() => (!err && resolve()) || reject(err), 100));
+			.end((err?: Error) => {
+				if (err) {
+					return reject(err);
+				}
+
+				if (debounce) {
+					setTimeout(resolve, 100);
+					return;
+				}
+
+				resolve();
+			});
 	});
 
 export const getSettingValueById = async (setting: string): Promise<ISetting['value']> => {

apps/meteor/tests/end-to-end/api/federation.ts

@@ -7,15 +7,17 @@ import { updateSetting } from '../../data/permissions.helper';
 describe('federation', () => {
 	before((done) => getCredentials(done));
 
+	// FIXME: why debouncing is causing timeouts here on the hooks?
+	// Since we don't care about the watchers on this setting, not debouncing is fine.
 	describe('well-known', () => {
 		describe('when matrix disabled', () => {
 			before(async () => {
-				await updateSetting('Federation_Matrix_enabled', false);
-				await updateSetting('Federation_Matrix_serve_well_known', true);
+				await updateSetting('Federation_Matrix_enabled', false, false);
+				await updateSetting('Federation_Matrix_serve_well_known', true, false);
 			});
 
 			after(async () => {
-				await updateSetting('Federation_Matrix_serve_well_known', false);
+				await updateSetting('Federation_Matrix_serve_well_known', false, false);
 			});
 
 			it('should return 404 not found', async () => {
@@ -27,12 +29,12 @@ describe('federation', () => {
 
 		describe('when matrix enabled but well-known disabled', () => {
 			before(async () => {
-				await updateSetting('Federation_Matrix_enabled', true);
-				await updateSetting('Federation_Matrix_serve_well_known', false);
+				await updateSetting('Federation_Matrix_enabled', true, false);
+				await updateSetting('Federation_Matrix_serve_well_known', false, false);
 			});
 
 			after(async () => {
-				await updateSetting('Federation_Matrix_enabled', false);
+				await updateSetting('Federation_Matrix_enabled', false, false);
 			});
 
 			it('should return 404 not found', async () => {
@@ -44,13 +46,13 @@ describe('federation', () => {
 
 		describe('when enabled', () => {
 			before(async () => {
-				await updateSetting('Federation_Matrix_enabled', true);
-				await updateSetting('Federation_Matrix_serve_well_known', true);
+				await updateSetting('Federation_Matrix_enabled', true, false);
+				await updateSetting('Federation_Matrix_serve_well_known', true, false);
 			});
 
 			after(async () => {
-				await updateSetting('Federation_Matrix_enabled', false);
-				await updateSetting('Federation_Matrix_serve_well_known', false);
+				await updateSetting('Federation_Matrix_enabled', false, false);
+				await updateSetting('Federation_Matrix_serve_well_known', false, false);
 			});
 
 			it('should return matrix information', async () => {

package.json

@@ -63,7 +63,6 @@
 		"underscore": "1.13.7",
 		"lodash": "4.17.21",
 		"mongodb@^4.17.1": "patch:mongodb@npm:4.17.1#.yarn/patches/mongodb-npm-4.17.1-a2fe811ff1.patch",
-		"@rocket.chat/forked-matrix-sdk-crypto-nodejs": "0.1.0-beta.13",
 		"typia@~6.9.0": "patch:typia@npm%3A6.9.0#./.yarn/patches/typia-npm-6.9.0-2fd4d85f25.patch",
 		"moleculer@^0.14.34": "patch:moleculer@npm%3A0.14.34#./.yarn/patches/moleculer-npm-0.14.34-440e26767d.patch",
 		"mongodb@npm:^4.3.1": "patch:mongodb@npm%3A4.17.2#~/.yarn/patches/mongodb-npm-4.17.2-40d1286d70.patch"