[WIP] Inline widgets

package.json

@@ -65,6 +65,7 @@
     "counterpart": "^0.18.6",
     "diff-dom": "^4.2.2",
     "diff-match-patch": "^1.0.5",
+    "embed-video": "^2.0.4",
     "emojibase-data": "^5.1.1",
     "emojibase-regex": "^4.1.1",
     "escape-html": "^1.0.3",
@@ -77,6 +78,7 @@
     "highlight.js": "^10.5.0",
     "html-entities": "^1.4.0",
     "is-ip": "^3.1.0",
+    "jquery": "^3.6.0",
     "katex": "^0.12.0",
     "linkifyjs": "^2.1.9",
     "lodash": "^4.17.20",

res/css/_components.scss

@@ -111,8 +111,8 @@
 @import "./views/elements/_AddressSelector.scss";
 @import "./views/elements/_AddressTile.scss";
 @import "./views/elements/_DesktopBuildsNotice.scss";
-@import "./views/elements/_DirectorySearchBox.scss";
 @import "./views/elements/_DesktopCapturerSourcePicker.scss";
+@import "./views/elements/_DirectorySearchBox.scss";
 @import "./views/elements/_Dropdown.scss";
 @import "./views/elements/_EditableItemList.scss";
 @import "./views/elements/_ErrorBoundary.scss";
@@ -159,6 +159,7 @@
 @import "./views/messages/_MStickerBody.scss";
 @import "./views/messages/_MTextBody.scss";
 @import "./views/messages/_MVideoBody.scss";
+@import "./views/messages/_MWidgetBody.scss";
 @import "./views/messages/_MessageActionBar.scss";
 @import "./views/messages/_MessageTimestamp.scss";
 @import "./views/messages/_MjolnirBody.scss";
@@ -217,14 +218,14 @@
 @import "./views/settings/_DevicesPanel.scss";
 @import "./views/settings/_E2eAdvancedPanel.scss";
 @import "./views/settings/_EmailAddresses.scss";
-@import "./views/settings/_SpellCheckLanguages.scss";
 @import "./views/settings/_IntegrationManager.scss";
 @import "./views/settings/_Notifications.scss";
 @import "./views/settings/_PhoneNumbers.scss";
 @import "./views/settings/_ProfileSettings.scss";
 @import "./views/settings/_SecureBackupPanel.scss";
 @import "./views/settings/_SetIdServer.scss";
 @import "./views/settings/_SetIntegrationManager.scss";
+@import "./views/settings/_SpellCheckLanguages.scss";
 @import "./views/settings/_UpdateCheckButton.scss";
 @import "./views/settings/tabs/_SettingsTab.scss";
 @import "./views/settings/tabs/room/_GeneralRoomSettingsTab.scss";

res/css/views/messages/_MWidgetBody.scss

@@ -0,0 +1,24 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+.mx_MWidgetBody {
+    height: 250px;
+    width: 400px;
+
+    .mx_AppTileFullWidth {
+        height: 100%;
+    }
+}

src/components/views/elements/AppTile.js

@@ -322,8 +322,15 @@ export default class AppTile extends React.Component {
         // this would only be for content hosted on the same origin as the element client: anything
         // hosted on the same origin as the client will get the same access as if you clicked
         // a link to it.
-        const sandboxFlags = "allow-forms allow-popups allow-popups-to-escape-sandbox "+
+        let sandboxFlags = "allow-forms allow-popups allow-popups-to-escape-sandbox "+
             "allow-same-origin allow-scripts allow-presentation";
+        if (this.props.strictSandbox)  {
+            // XXX: A compromised wrapped HTML widget can access localStorage with allow-same-origin.
+            // We already filter down the HTML though, so this should be fine. Just scary.
+            // Note: Removing allow-same-origin means that postMessage requests from the frame get
+            // an `origin` with the literal string "null" and no way to deduce it.
+            sandboxFlags = "allow-forms allow-scripts allow-presentation allow-same-origin";
+        }
 
         // Additional iframe feature pemissions
         // (see - https://sites.google.com/a/chromium.org/dev/Home/chromium-security/deprecating-permissions-in-cross-origin-iframes and https://wicg.github.io/feature-policy/)
@@ -485,6 +492,9 @@ AppTile.propTypes = {
     userWidget: PropTypes.bool,
     // sets the pointer-events property on the iframe
     pointerEvents: PropTypes.string,
+    // If true, the sandbox prevents most things. Intended for inline widgets
+    // with untrusted HTML.
+    strictSandbox: PropTypes.bool,
 };
 
 AppTile.defaultProps = {

src/components/views/messages/MWidgetBody.tsx

@@ -0,0 +1,100 @@
+/*
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import WidgetUtils from "../../../utils/WidgetUtils";
+import * as React from 'react';
+import * as sdk from "../../../index";
+import AppTile from "../elements/AppTile";
+import {MatrixClientPeg} from "../../../MatrixClientPeg";
+import {IApp} from "../../../stores/WidgetStore";
+
+export default class MWidgetBody extends React.Component<any, any> {
+    // static propTypes: {
+    //     mxEvent: PropTypes.object.isRequired, // MatrixEvent
+    //
+    //     // Passthroughs for TextualBody
+    //     highlights: PropTypes.array,
+    //     highlightLink: PropTypes.string,
+    //     showUrlPreview: PropTypes.bool,
+    //     onHeightChanged: PropTypes.func,
+    //     tileShape: PropTypes.string,
+    // };
+
+    renderAsText() {
+        const TextualBody = sdk.getComponent("messages.TextualBody");
+        return <TextualBody
+            mxEvent={this.props.mxEvent}
+            highlights={this.props.highlights}
+            highlightLink={this.props.highlightLink}
+            showUrlPreview={this.props.showUrlPreview}
+            onHeightChanged={this.props.onHeightChanged}
+            tileShapse={this.props.tileShape}
+        />;
+    }
+
+    render() {
+        const widgetInfo = this.props.mxEvent.getContent();
+
+        let widgetUrl = widgetInfo['widget_url'];
+        let wantedCapabilities = [];
+        let showTitle = false;
+        let strictFrame = false;
+        if (!widgetUrl) {
+            const widgetHtml = widgetInfo['widget_html'];
+            if (!widgetHtml) return this.renderAsText();
+
+            const info = WidgetUtils.wrapWidgetHtml(widgetHtml);
+            widgetUrl = info.url;
+            wantedCapabilities = info.wantedCapabilities;
+            showTitle = true;
+            strictFrame = true;
+        }
+        if (!widgetUrl) return this.renderAsText();
+
+        const app: IApp = {
+            ...widgetInfo,
+            // XXX: Is this a secure enough widget ID?
+            id: this.props.mxEvent.getRoomId() + "_" + this.props.mxEvent.getId(),
+            url: widgetUrl,
+        };
+
+        return <div className="mx_MWidgetBody">
+            <AppTile
+                id={app.id}
+                url={widgetUrl}
+                name={widgetInfo['name'] || "Widget"}
+                room={MatrixClientPeg.get().getRoom(this.props.mxEvent.getRoomId())}
+                type={widgetInfo['type'] || "m.custom"}
+                app={app}
+                fullWidth={true}
+                userId={MatrixClientPeg.get().credentials.userId}
+                creatorUserId={this.props.mxEvent.getSender()}
+                waitForIframeLoad={true}
+                show={true}
+                showMenubar={showTitle}
+                showTitle={showTitle}
+                showMinimise={false}
+                showDelete={false}
+                showCancel={false}
+                showPopout={false}
+                widgetPageTitle={(widgetInfo['data'] && widgetInfo['data']['title']) ? widgetInfo['data']['title'] : null}
+                handleMinimisePointerEvents={false}
+                whitelistCapabilities={wantedCapabilities}
+                strictSandbox={strictFrame}
+            />
+        </div>;
+    };
+}

src/components/views/messages/MessageEvent.js

@@ -71,6 +71,9 @@ export default class MessageEvent extends React.Component {
             'm.file': sdk.getComponent('messages.MFileBody'),
             'm.audio': sdk.getComponent('messages.MAudioBody'),
             'm.video': sdk.getComponent('messages.MVideoBody'),
+            'm.widget': SettingsStore.getValue("feature_inline_widgets")
+                ? sdk.getComponent('messages.MWidgetBody')
+                : sdk.getComponent('messages.TextualBody'),
         };
         const evTypes = {
             'm.sticker': sdk.getComponent('messages.MStickerBody'),

src/components/views/rooms/SendMessageComposer.js

@@ -40,11 +40,12 @@ import {_t, _td} from '../../../languageHandler';
 import ContentMessages from '../../../ContentMessages';
 import {Key, isOnlyCtrlOrCmdKeyEvent} from "../../../Keyboard";
 import MatrixClientContext from "../../../contexts/MatrixClientContext";
+import WidgetUtils from "../../../utils/WidgetUtils";
+import SettingsStore from "../../../settings/SettingsStore";
 import RateLimitedFunc from '../../../ratelimitedfunc';
 import {Action} from "../../../dispatcher/actions";
 import {containsEmoji} from "../../../effects/utils";
 import {CHAT_EFFECTS} from '../../../effects';
-import SettingsStore from "../../../settings/SettingsStore";
 import CountlyAnalytics from "../../../CountlyAnalytics";
 import {MatrixClientPeg} from "../../../MatrixClientPeg";
 import EMOJI_REGEX from 'emojibase-regex';
@@ -335,6 +336,15 @@ export default class SendMessageComposer extends React.Component {
 
         let shouldSend = true;
 
+        const inlineWidget = startsWith(this.model, "https://")
+            ? WidgetUtils.tryConvertInputToInlineWidget(textSerialize(this.model))
+            : null;
+        if (inlineWidget && SettingsStore.getValue("feature_inline_widgets")) {
+            console.log("Message can be an inline widget - sending widget");
+            this.context.sendMessage(this.props.room.roomId, inlineWidget);
+            shouldSend = false;
+        }
+
         if (!containsEmote(this.model) && this._isSlashCommand()) {
             const [cmd, commandText] = this._getSlashCommand();
             if (cmd) {

src/i18n/strings/en_EN.json

@@ -784,6 +784,7 @@
     "Change notification settings": "Change notification settings",
     "Spaces prototype. Incompatible with Communities, Communities v2 and Custom Tags. Requires compatible homeserver for some features.": "Spaces prototype. Incompatible with Communities, Communities v2 and Custom Tags. Requires compatible homeserver for some features.",
     "Render LaTeX maths in messages": "Render LaTeX maths in messages",
+    "Widgets in the timeline": "Widgets in the timeline",
     "Communities v2 prototypes. Requires compatible homeserver. Highly experimental - use with caution.": "Communities v2 prototypes. Requires compatible homeserver. Highly experimental - use with caution.",
     "New spinner design": "New spinner design",
     "Message Pinning": "Message Pinning",

src/settings/Settings.ts

@@ -134,6 +134,12 @@ export const SETTINGS: {[setting: string]: ISetting} = {
         supportedLevels: LEVELS_FEATURE,
         default: false,
     },
+    "feature_inline_widgets": {
+        isFeature: true,
+        displayName: _td("Widgets in the timeline"),
+        supportedLevels: LEVELS_FEATURE,
+        default: false,
+    },
     "feature_communities_v2_prototypes": {
         isFeature: true,
         displayName: _td(

src/utils/WidgetUtils.ts

@@ -29,6 +29,8 @@ import {objectClone} from "./objects";
 import {_t} from "../languageHandler";
 import {Capability, IWidget, IWidgetData, MatrixCapabilities} from "matrix-widget-api";
 import {IApp} from "../stores/WidgetStore";
+import embedVideo from "embed-video";
+import $ from "jquery";
 
 // How long we wait for the state event echo to come back from the server
 // before waitFor[Room/User]Widget rejects its promise
@@ -44,6 +46,65 @@ export interface IWidgetEvent {
 }
 
 export default class WidgetUtils {
+    /**
+     * Tries to convert a given input string into an inline widget. If the
+     * input string does not translate to an inline widget, a falsey value
+     * is returned. Otherwise, the content object for an m.room.message
+     * representing the inline widget is returned.
+     * @param {string} inputText The input text to try and parse.
+     * @returns {*} The m.room.message content object or a falsey value.
+     */
+    static tryConvertInputToInlineWidget(inputText) {
+        // Dev note: borrowed from Dimension with permission
+        // https://github.com/turt2live/matrix-dimension/blob/f773b7a3ae4af4a6929d58c0528574f6b240d574/web/app/configs/widget/youtube/youtube.widget.component.ts#L59-L70
+        const embedCode = embedVideo(inputText);
+        if (!embedCode) return null; // can't be converted
+        // HACK: Grab the video URL from the iframe embed code
+        const videoUrl = "https:" + $(embedCode).attr("src");
+
+        return {
+            msgtype: "m.widget",
+            body: inputText,
+            type: "m.custom",
+            widget_url: videoUrl,
+        };
+    }
+
+    /**
+     * Sanitizes and wraps a block of HTML for a local widget.
+     * @param {string} html The widget's HTML.
+     * @returns {{wantedCapabilities: string[], url: string}} An object containing
+     * the URL for the wrapped HTML and the capabilities the widget wants.
+     */
+    static wrapWidgetHtml(html) {
+        const wantedCapabilities = [];
+
+        // TODO: Sanitize HTML
+        // TODO: Parse capabilities, etc from sanitized HTML
+
+        // HACK: Temporary measure
+        wantedCapabilities.push("m.send.m.room.message");
+        wantedCapabilities.push("m.send.m.room.hidden");
+
+        const wrapperOpts = {
+            html: html,
+            capabilities: wantedCapabilities,
+        };
+
+        const base64WrapperOpts = btoa(JSON.stringify(wrapperOpts))
+            .replace(/\+/g, '-')
+            .replace(/\//g, '_');
+
+        const loc = window.location;
+        let urlBase = `${loc.protocol}//${loc.host}${loc.pathname}`;
+        if (!urlBase.endsWith("/")) urlBase = `${urlBase}/`;
+
+        return {
+            url: `${urlBase}inline_widget_wrapper/index.html#${base64WrapperOpts}`,
+            wantedCapabilities: wantedCapabilities,
+        };
+    }
+
     /* Returns true if user is able to send state events to modify widgets in this room
      * (Does not apply to non-room-based / user widgets)
      * @param roomId -- The ID of the room to check