Merge pull request #5955 from matrix-org/travis/voicemessages/timeline

Early rendering for voice messages in the timeline

res/css/_components.scss

@@ -161,6 +161,7 @@
 @import "./views/messages/_MStickerBody.scss";
 @import "./views/messages/_MTextBody.scss";
 @import "./views/messages/_MVideoBody.scss";
+@import "./views/messages/_MVoiceMessageBody.scss";
 @import "./views/messages/_MessageActionBar.scss";
 @import "./views/messages/_MessageTimestamp.scss";
 @import "./views/messages/_MjolnirBody.scss";

res/css/views/messages/_MVoiceMessageBody.scss

@@ -0,0 +1,19 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+.mx_MVoiceMessageBody {
+    display: inline-block; // makes the playback controls magically line up
+}

src/components/views/messages/MVoiceMessageBody.tsx

@@ -0,0 +1,106 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from "react";
+import {MatrixEvent} from "matrix-js-sdk/src/models/event";
+import {replaceableComponent} from "../../../utils/replaceableComponent";
+import {Playback} from "../../../voice/Playback";
+import MFileBody from "./MFileBody";
+import InlineSpinner from '../elements/InlineSpinner';
+import {_t} from "../../../languageHandler";
+import {mediaFromContent} from "../../../customisations/Media";
+import {decryptFile} from "../../../utils/DecryptFile";
+import RecordingPlayback from "../voice_messages/RecordingPlayback";
+
+interface IProps {
+    mxEvent: MatrixEvent;
+}
+
+interface IState {
+    error?: Error;
+    playback?: Playback;
+    decryptedBlob?: Blob;
+}
+
+@replaceableComponent("views.messages.MVoiceMessageBody")
+export default class MVoiceMessageBody extends React.PureComponent<IProps, IState> {
+    constructor(props: IProps) {
+        super(props);
+
+        this.state = {};
+    }
+
+    public async componentDidMount() {
+        let buffer: ArrayBuffer;
+        const content = this.props.mxEvent.getContent();
+        const media = mediaFromContent(content);
+        if (media.isEncrypted) {
+            try {
+                const blob = await decryptFile(content.file);
+                buffer = await blob.arrayBuffer();
+                this.setState({decryptedBlob: blob});
+            } catch (e) {
+                this.setState({error: e});
+                console.warn("Unable to decrypt voice message", e);
+                return; // stop processing the audio file
+            }
+        } else {
+            try {
+                buffer = await media.downloadSource().then(r => r.blob()).then(r => r.arrayBuffer());
+            } catch (e) {
+                this.setState({error: e});
+                console.warn("Unable to download voice message", e);
+                return; // stop processing the audio file
+            }
+        }
+
+        const waveform = content?.["org.matrix.msc1767.audio"]?.waveform?.map(p => p / 1024);
+
+        // We should have a buffer to work with now: let's set it up
+        const playback = new Playback(buffer, waveform);
+        this.setState({playback});
+        // Note: the RecordingPlayback component will handle preparing the Playback class for us.
+    }
+
+    public render() {
+        if (this.state.error) {
+            // TODO: @@TR: Verify error state
+            return (
+                <span className="mx_MVoiceMessageBody">
+                    <img src={require("../../../../res/img/warning.svg")} width="16" height="16" />
+                    { _t("Error processing voice message") }
+                </span>
+            );
+        }
+
+        if (!this.state.playback) {
+            // TODO: @@TR: Verify loading/decrypting state
+            return (
+                <span className="mx_MVoiceMessageBody">
+                    <InlineSpinner />
+                </span>
+            );
+        }
+
+        // At this point we should have a playable state
+        return (
+            <span className="mx_MVoiceMessageBody">
+                <RecordingPlayback playback={this.state.playback} />
+                <MFileBody {...this.props} decryptedBlob={this.state.decryptedBlob} showGenericPlaceholder={false} />
+            </span>
+        )
+    }
+}

src/components/views/messages/MVoiceOrAudioBody.tsx

@@ -0,0 +1,39 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from "react";
+import {MatrixEvent} from "matrix-js-sdk/src/models/event";
+import MAudioBody from "./MAudioBody";
+import {replaceableComponent} from "../../../utils/replaceableComponent";
+import SettingsStore from "../../../settings/SettingsStore";
+import MVoiceMessageBody from "./MVoiceMessageBody";
+
+interface IProps {
+    mxEvent: MatrixEvent;
+}
+
+@replaceableComponent("views.messages.MVoiceOrAudioBody")
+export default class MVoiceOrAudioBody extends React.PureComponent<IProps> {
+    public render() {
+        const isVoiceMessage = !!this.props.mxEvent.getContent()['org.matrix.msc2516.voice'];
+        const voiceMessagesEnabled = SettingsStore.getValue("feature_voice_messages");
+        if (isVoiceMessage && voiceMessagesEnabled) {
+            return <MVoiceMessageBody {...this.props} />;
+        } else {
+            return <MAudioBody {...this.props} />;
+        }
+    }
+}

src/components/views/messages/MessageEvent.js

@@ -72,12 +72,8 @@ export default class MessageEvent extends React.Component {
             'm.emote': sdk.getComponent('messages.TextualBody'),
             'm.image': sdk.getComponent('messages.MImageBody'),
             'm.file': sdk.getComponent('messages.MFileBody'),
-            'm.audio': sdk.getComponent('messages.MAudioBody'),
+            'm.audio': sdk.getComponent('messages.MVoiceOrAudioBody'),
             'm.video': sdk.getComponent('messages.MVideoBody'),
-
-            // TODO: @@ TravisR: Use labs flag determination.
-            // MSC: https://github.com/matrix-org/matrix-doc/pull/2516
-            'org.matrix.msc2516.voice': sdk.getComponent('messages.MAudioBody'),
         };
         const evTypes = {
             'm.sticker': sdk.getComponent('messages.MStickerBody'),

src/components/views/rooms/VoiceRecordComposerTile.tsx

@@ -27,6 +27,7 @@ import LiveRecordingClock from "../voice_messages/LiveRecordingClock";
 import {VoiceRecordingStore} from "../../../stores/VoiceRecordingStore";
 import {UPDATE_EVENT} from "../../../stores/AsyncStore";
 import RecordingPlayback from "../voice_messages/RecordingPlayback";
+import {MsgType} from "matrix-js-sdk/src/@types/event";
 import Modal from "../../../Modal";
 import ErrorDialog from "../dialogs/ErrorDialog";
 import CallMediaHandler from "../../../CallMediaHandler";
@@ -67,8 +68,8 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
         const mxc = await this.state.recorder.upload();
         MatrixClientPeg.get().sendMessage(this.props.room.roomId, {
             "body": "Voice message",
-            "msgtype": "org.matrix.msc2516.voice",
-            //"msgtype": MsgType.Audio,
+            //"msgtype": "org.matrix.msc2516.voice",
+            "msgtype": MsgType.Audio,
             "url": mxc,
             "info": {
                 duration: Math.round(this.state.recorder.durationSeconds * 1000),
@@ -86,10 +87,6 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
             },
             "org.matrix.msc1767.audio": {
                 duration: Math.round(this.state.recorder.durationSeconds * 1000),
-                // TODO: @@ TravisR: Waveform? (MSC1767 decision)
-            },
-            "org.matrix.experimental.msc2516.voice": { // MSC2516+MSC1767 experiment
-                duration: Math.round(this.state.recorder.durationSeconds * 1000),
 
                 // Events can't have floats, so we try to maintain resolution by using 1024
                 // as a maximum value. The waveform contains values between zero and 1, so this
@@ -98,6 +95,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
                 // We're expecting about one data point per second of audio.
                 waveform: this.state.recorder.getPlayback().waveform.map(v => Math.round(v * 1024)),
             },
+            "org.matrix.msc2516.voice": {}, // No content, this is a rendering hint
         });
         await this.disposeRecording();
     }

src/i18n/strings/en_EN.json

@@ -1852,6 +1852,7 @@
     "%(name)s wants to verify": "%(name)s wants to verify",
     "You sent a verification request": "You sent a verification request",
     "Error decrypting video": "Error decrypting video",
+    "Error processing voice message": "Error processing voice message",
     "Show all": "Show all",
     "Reactions": "Reactions",
     "<reactors/><reactedWith> reacted with %(content)s</reactedWith>": "<reactors/><reactedWith> reacted with %(content)s</reactedWith>",

src/voice/Playback.ts

@@ -50,7 +50,7 @@ export class Playback extends EventEmitter implements IDestroyable {
     constructor(private buf: ArrayBuffer, seedWaveform = DEFAULT_WAVEFORM) {
         super();
         this.context = new AudioContext();
-        this.resampledWaveform = arrayFastResample(seedWaveform, PLAYBACK_WAVEFORM_SAMPLES);
+        this.resampledWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, PLAYBACK_WAVEFORM_SAMPLES);
         this.waveformObservable.update(this.resampledWaveform);
         this.clock = new PlaybackClock(this.context);
     }