Merge pull request #6240 from matrix-org/gsouquet/voice-messages-waveform-perf

Improve audio recording performance

res/css/views/voice_messages/_PlaybackContainer.scss

@@ -33,9 +33,14 @@ limitations under the License.
     font-size: $font-14px;
     line-height: $font-24px;
 
+    contain: content;
+
     .mx_Waveform {
         .mx_Waveform_bar {
             background-color: $voice-record-waveform-incomplete-fg-color;
+            height: 100%;
+            /* Variable set by a JS component */
+            transform: scaleY(max(0.05, var(--barHeight)));
 
             &.mx_Waveform_bar_100pct {
                 // Small animation to remove the mechanical feel of progress

src/components/views/rooms/VoiceRecordComposerTile.tsx

@@ -15,19 +15,26 @@ limitations under the License.
 */
 
 import AccessibleTooltipButton from "../elements/AccessibleTooltipButton";
-import {_t} from "../../../languageHandler";
+import { _t } from "../../../languageHandler";
 import React, {ReactNode} from "react";
-import {RecordingState, VoiceRecording} from "../../../voice/VoiceRecording";
+import {
+    IRecordingUpdate,
+    RECORDING_PLAYBACK_SAMPLES,
+    RecordingState,
+    VoiceRecording,
+} from "../../../voice/VoiceRecording";
 import {Room} from "matrix-js-sdk/src/models/room";
-import {MatrixClientPeg} from "../../../MatrixClientPeg";
+import { MatrixClientPeg } from "../../../MatrixClientPeg";
 import classNames from "classnames";
 import LiveRecordingWaveform from "../voice_messages/LiveRecordingWaveform";
-import {replaceableComponent} from "../../../utils/replaceableComponent";
+import { replaceableComponent } from "../../../utils/replaceableComponent";
+import { arrayFastResample, arraySeed } from "../../../utils/arrays";
+import { percentageOf } from "../../../utils/numbers";
 import LiveRecordingClock from "../voice_messages/LiveRecordingClock";
-import {VoiceRecordingStore} from "../../../stores/VoiceRecordingStore";
+import { VoiceRecordingStore } from "../../../stores/VoiceRecordingStore";
 import {UPDATE_EVENT} from "../../../stores/AsyncStore";
 import RecordingPlayback from "../voice_messages/RecordingPlayback";
-import {MsgType} from "matrix-js-sdk/src/@types/event";
+import { MsgType } from "matrix-js-sdk/src/@types/event";
 import Modal from "../../../Modal";
 import ErrorDialog from "../dialogs/ErrorDialog";
 import MediaDeviceHandler from "../../../MediaDeviceHandler";
@@ -39,6 +46,8 @@ interface IProps {
 interface IState {
     recorder?: VoiceRecording;
     recordingPhase?: RecordingState;
+    relHeights: number[];
+    seconds: number;
 }
 
 /**
@@ -46,18 +55,58 @@ interface IState {
  */
 @replaceableComponent("views.rooms.VoiceRecordComposerTile")
 export default class VoiceRecordComposerTile extends React.PureComponent<IProps, IState> {
+    private waveform: number[] = [];
+    private seconds = 0;
+    private scheduledAnimationFrame = false;
+
     public constructor(props) {
         super(props);
 
         this.state = {
             recorder: null, // no recording started by default
+            seconds: 0,
+            relHeights: arraySeed(0, RECORDING_PLAYBACK_SAMPLES),
         };
     }
 
+    public componentDidUpdate(prevProps, prevState) {
+        if (!prevState.recorder && this.state.recorder) {
+            this.state.recorder.liveData.onUpdate(this.onRecordingUpdate);
+        }
+    }
+
     public async componentWillUnmount() {
         await VoiceRecordingStore.instance.disposeRecording();
     }
 
+    private onRecordingUpdate = (update: IRecordingUpdate): void => {
+        this.waveform = update.waveform;
+        this.seconds = update.timeSeconds;
+
+        if (this.scheduledAnimationFrame) {
+            return;
+        }
+
+        this.scheduledAnimationFrame = true;
+        // The audio recorder flushes data faster than the screen refresh rate
+        // Using requestAnimationFrame makes sure that we only flush the data
+        // to react once per tick to avoid unneeded work.
+        requestAnimationFrame(() => {
+            // The waveform and the downsample target are pretty close, so we should be fine to
+            // do this, despite the docs on arrayFastResample.
+            const bars = arrayFastResample(Array.from(this.waveform), RECORDING_PLAYBACK_SAMPLES);
+            this.setState({
+                // The incoming data is between zero and one, but typically even screaming into a
+                // microphone won't send you over 0.6, so we artificially adjust the gain for the
+                // waveform. This results in a slightly more cinematic/animated waveform for the
+                // user.
+                relHeights: bars.map(b => percentageOf(b, 0, 0.50)),
+                seconds: this.seconds,
+            });
+            this.scheduledAnimationFrame = false;
+        });
+    }
+
     // called by composer
     public async send() {
         if (!this.state.recorder) {

src/components/views/voice_messages/Clock.tsx

@@ -15,9 +15,9 @@ limitations under the License.
 */
 
 import React from "react";
-import {replaceableComponent} from "../../../utils/replaceableComponent";
+import { replaceableComponent } from "../../../utils/replaceableComponent";
 
-interface IProps {
+export interface IProps {
     seconds: number;
 }
 

src/components/views/voice_messages/LiveRecordingClock.tsx

@@ -1,12 +1,9 @@
 /*
 Copyright 2021 The Matrix.org Foundation C.I.C.
-
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
     http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -15,12 +12,16 @@ limitations under the License.
 */
 
 import React from "react";
-import {IRecordingUpdate, VoiceRecording} from "../../../voice/VoiceRecording";
-import {replaceableComponent} from "../../../utils/replaceableComponent";
 import Clock from "./Clock";
+import { replaceableComponent } from "../../../utils/replaceableComponent";
+import { MarkedExecution } from "../../../utils/MarkedExecution";
+import {
+    IRecordingUpdate,
+    VoiceRecording,
+} from "../../../voice/VoiceRecording";
 
 interface IProps {
-    recorder: VoiceRecording;
+    recorder?: VoiceRecording;
 }
 
 interface IState {
@@ -32,16 +33,31 @@ interface IState {
  */
 @replaceableComponent("views.voice_messages.LiveRecordingClock")
 export default class LiveRecordingClock extends React.PureComponent<IProps, IState> {
-    public constructor(props) {
-        super(props);
+    private seconds = 0;
+    private scheduledUpdate = new MarkedExecution(
+        () => this.updateClock(),
+        () => requestAnimationFrame(() => this.scheduledUpdate.trigger()),
+    );
 
-        this.state = {seconds: 0};
-        this.props.recorder.liveData.onUpdate(this.onRecordingUpdate);
+    constructor(props) {
+        super(props);
+        this.state = {
+            seconds: 0,
+        };
     }
 
-    private onRecordingUpdate = (update: IRecordingUpdate) => {
-        this.setState({seconds: update.timeSeconds});
-    };
+    componentDidMount() {
+        this.props.recorder.liveData.onUpdate((update: IRecordingUpdate) => {
+            this.seconds = update.timeSeconds;
+            this.scheduledUpdate.mark();
+        });
+    }
+
+    private updateClock() {
+        this.setState({
+            seconds: this.seconds,
+        });
+    }
 
     public render() {
         return <Clock seconds={this.state.seconds} />;

src/components/views/voice_messages/LiveRecordingWaveform.tsx

@@ -1,12 +1,9 @@
 /*
 Copyright 2021 The Matrix.org Foundation C.I.C.
-
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
     http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -15,18 +12,20 @@ limitations under the License.
 */
 
 import React from "react";
-import {IRecordingUpdate, RECORDING_PLAYBACK_SAMPLES, VoiceRecording} from "../../../voice/VoiceRecording";
-import {replaceableComponent} from "../../../utils/replaceableComponent";
-import {arrayFastResample, arraySeed} from "../../../utils/arrays";
-import {percentageOf} from "../../../utils/numbers";
 import Waveform from "./Waveform";
+import { replaceableComponent } from "../../../utils/replaceableComponent";
+import { MarkedExecution } from "../../../utils/MarkedExecution";
+import {
+    IRecordingUpdate,
+    VoiceRecording,
+} from "../../../voice/VoiceRecording";
 
 interface IProps {
-    recorder: VoiceRecording;
+    recorder?: VoiceRecording;
 }
 
 interface IState {
-    heights: number[];
+    waveform: number[];
 }
 
 /**
@@ -34,27 +33,37 @@ interface IState {
  */
 @replaceableComponent("views.voice_messages.LiveRecordingWaveform")
 export default class LiveRecordingWaveform extends React.PureComponent<IProps, IState> {
-    public constructor(props) {
-        super(props);
-
-        this.state = {heights: arraySeed(0, RECORDING_PLAYBACK_SAMPLES)};
-        this.props.recorder.liveData.onUpdate(this.onRecordingUpdate);
-    }
-
-    private onRecordingUpdate = (update: IRecordingUpdate) => {
-        // The waveform and the downsample target are pretty close, so we should be fine to
-        // do this, despite the docs on arrayFastResample.
-        const bars = arrayFastResample(Array.from(update.waveform), RECORDING_PLAYBACK_SAMPLES);
-        this.setState({
-            // The incoming data is between zero and one, but typically even screaming into a
-            // microphone won't send you over 0.6, so we artificially adjust the gain for the
-            // waveform. This results in a slightly more cinematic/animated waveform for the
-            // user.
-            heights: bars.map(b => percentageOf(b, 0, 0.50)),
-        });
+    public static defaultProps = {
+        progress: 1,
     };
 
+    private waveform: number[] = [];
+    private scheduledUpdate = new MarkedExecution(
+        () => this.updateWaveform(),
+        () => requestAnimationFrame(() => this.scheduledUpdate.trigger()),
+    );
+
+    constructor(props) {
+        super(props);
+        this.state = {
+            waveform: [],
+        };
+    }
+
+    componentDidMount() {
+        this.props.recorder.liveData.onUpdate((update: IRecordingUpdate) => {
+            this.waveform = update.waveform;
+            this.scheduledUpdate.mark();
+        });
+    }
+
+    private updateWaveform() {
+        this.setState({
+            waveform: this.waveform,
+        })
+    }
+
     public render() {
-        return <Waveform relHeights={this.state.heights} />;
+        return <Waveform relHeights={this.state.waveform} />;
     }
 }

src/components/views/voice_messages/Waveform.tsx

@@ -15,10 +15,10 @@ limitations under the License.
 */
 
 import React from "react";
-import {replaceableComponent} from "../../../utils/replaceableComponent";
+import { replaceableComponent } from "../../../utils/replaceableComponent";
 import classNames from "classnames";
 
-interface IProps {
+export interface IProps {
     relHeights: number[]; // relative heights (0-1)
     progress: number; // percent complete, 0-1, default 100%
 }
@@ -34,16 +34,19 @@ interface IState {
  * For CSS purposes, a mx_Waveform_bar_100pct class is added when the bar should be
  * "filled", as a demonstration of the progress property.
  */
+
+import { CSSProperties } from "react";
+
+export interface WaveformCSSProperties extends CSSProperties {
+    '--barHeight': number;
+}
+
 @replaceableComponent("views.voice_messages.Waveform")
 export default class Waveform extends React.PureComponent<IProps, IState> {
     public static defaultProps = {
         progress: 1,
     };
 
-    public constructor(props) {
-        super(props);
-    }
-
     public render() {
         return <div className='mx_Waveform'>
             {this.props.relHeights.map((h, i) => {
@@ -53,7 +56,9 @@ export default class Waveform extends React.PureComponent<IProps, IState> {
                     'mx_Waveform_bar': true,
                     'mx_Waveform_bar_100pct': isCompleteBar,
                 });
-                return <span key={i} style={{height: (h * 100) + '%'}} className={classes} />;
+                return <span key={i} style={{
+                    "--barHeight": h,
+                } as WaveformCSSProperties} className={classes} />;
             })}
         </div>;
     }

src/utils/MarkedExecution.ts

@@ -26,9 +26,11 @@ export class MarkedExecution {
 
     /**
      * Creates a MarkedExecution for the provided function.
-     * @param fn The function to be called upon trigger if marked.
+     * @param {Function} fn The function to be called upon trigger if marked.
+     * @param {Function} onMarkCallback A function that is called when a new mark is made. Not
+     * called if a mark is already flagged.
      */
-    constructor(private fn: () => void) {
+    constructor(private fn: () => void, private onMarkCallback?: () => void) {
     }
 
     /**
@@ -42,6 +44,7 @@ export class MarkedExecution {
      * Marks the function to be called upon trigger().
      */
     public mark() {
+        if (!this.marked) this.onMarkCallback?.();
         this.marked = true;
     }