From 730af94014ec14541358d725fc188e40267e79fe Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 30 Aug 2021 14:20:03 -0600 Subject: [PATCH] Autoplay semantics for voice messages Fixes https://github.com/vector-im/element-web/issues/18804 Fixes https://github.com/vector-im/element-web/issues/18715 Fixes https://github.com/vector-im/element-web/issues/18714 Fixes https://github.com/vector-im/element-web/issues/17961 --- src/audio/ManagedPlayback.ts | 2 +- src/audio/Playback.ts | 7 +- src/audio/PlaybackClock.ts | 6 +- src/audio/PlaybackManager.ts | 12 +- src/audio/PlaybackQueue.ts | 212 ++++++++++++++++++ src/components/views/messages/MAudioBody.tsx | 6 + .../views/rooms/VoiceRecordComposerTile.tsx | 2 +- 7 files changed, 236 insertions(+), 11 deletions(-) create mode 100644 src/audio/PlaybackQueue.ts diff --git a/src/audio/ManagedPlayback.ts b/src/audio/ManagedPlayback.ts index bff6ce7088..5db07671f1 100644 --- a/src/audio/ManagedPlayback.ts +++ b/src/audio/ManagedPlayback.ts @@ -26,7 +26,7 @@ export class ManagedPlayback extends Playback { } public async play(): Promise { - this.manager.playOnly(this); + this.manager.pauseAllExcept(this); return super.play(); } diff --git a/src/audio/Playback.ts b/src/audio/Playback.ts index 9dad828a79..03f3bad760 100644 --- a/src/audio/Playback.ts +++ b/src/audio/Playback.ts @@ -117,6 +117,8 @@ export class Playback extends EventEmitter implements IDestroyable { } public destroy() { + // Dev note: It's critical that we call stop() during cleanup to ensure that downstream callers + // are aware of the final clock position before the user triggered an unload. // noinspection JSIgnoredPromiseFromCall - not concerned about being called async here this.stop(); this.removeAllListeners(); @@ -177,9 +179,12 @@ export class Playback extends EventEmitter implements IDestroyable { this.waveformObservable.update(this.resampledWaveform); - this.emit(PlaybackState.Stopped); // signal that we're not decoding anymore this.clock.flagLoadTime(); // must happen first because setting the duration fires a clock update this.clock.durationSeconds = this.element ? this.element.duration : this.audioBuf.duration; + + // Signal that we're not decoding anymore. This is done last to ensure the clock is updated for + // when the downstream callers try to use it. + this.emit(PlaybackState.Stopped); // signal that we're not decoding anymore } private onPlaybackEnd = async () => { diff --git a/src/audio/PlaybackClock.ts b/src/audio/PlaybackClock.ts index 712d1bfa94..5716d6ac2f 100644 --- a/src/audio/PlaybackClock.ts +++ b/src/audio/PlaybackClock.ts @@ -89,9 +89,9 @@ export class PlaybackClock implements IDestroyable { return this.observable; } - private checkTime = () => { + private checkTime = (force = false) => { const now = this.timeSeconds; // calculated dynamically - if (this.lastCheck !== now) { + if (this.lastCheck !== now || force) { this.observable.update([now, this.durationSeconds]); this.lastCheck = now; } @@ -141,7 +141,7 @@ export class PlaybackClock implements IDestroyable { public syncTo(contextTime: number, clipTime: number) { this.clipStart = contextTime - clipTime; this.stopped = false; // count as a mid-stream pause (if we were stopped) - this.checkTime(); + this.checkTime(true); } public destroy() { diff --git a/src/audio/PlaybackManager.ts b/src/audio/PlaybackManager.ts index 58fa61df56..58c0b9b624 100644 --- a/src/audio/PlaybackManager.ts +++ b/src/audio/PlaybackManager.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { DEFAULT_WAVEFORM, Playback } from "./Playback"; +import { DEFAULT_WAVEFORM, Playback, PlaybackState } from "./Playback"; import { ManagedPlayback } from "./ManagedPlayback"; /** @@ -34,12 +34,14 @@ export class PlaybackManager { } /** - * Stops all other playback instances. If no playback is provided, all instances - * are stopped. + * Pauses all other playback instances. If no playback is provided, all playing + * instances are paused. * @param playback Optional. The playback to leave untouched. */ - public playOnly(playback?: Playback) { - this.instances.filter(p => p !== playback).forEach(p => p.stop()); + public pauseAllExcept(playback?: Playback) { + this.instances + .filter(p => p !== playback && p.currentState === PlaybackState.Playing) + .forEach(p => p.pause()); } public destroyPlaybackInstance(playback: ManagedPlayback) { diff --git a/src/audio/PlaybackQueue.ts b/src/audio/PlaybackQueue.ts new file mode 100644 index 0000000000..6df4c24897 --- /dev/null +++ b/src/audio/PlaybackQueue.ts @@ -0,0 +1,212 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { MatrixClient, MatrixEvent, Room } from "matrix-js-sdk"; +import { Playback, PlaybackState } from "./Playback"; +import { UPDATE_EVENT } from "../stores/AsyncStore"; +import { MatrixClientPeg } from "../MatrixClientPeg"; +import { arrayFastClone } from "../utils/arrays"; +import { PlaybackManager } from "./PlaybackManager"; +import { isVoiceMessage } from "../utils/EventUtils"; +import RoomViewStore from "../stores/RoomViewStore"; + +/** + * Audio playback queue management for a given room. This keeps track of where the user + * was at for each playback, what order the playbacks were played in, and triggers subsequent + * playbacks. + * + * Currently this is only intended to be used by voice messages. + * + * The primary mechanics are: + * * Persisted clock state for each playback instance (tied to Event ID). + * * Limited memory of playback order (see code; not persisted). + * * Autoplay of next eligible playback instance. + */ +export class PlaybackQueue { + private static queues = new Map(); // keyed by room ID + + private playbacks = new Map(); // keyed by event ID + private clockStates = new Map(); // keyed by event ID + private playbackIdOrder: string[] = []; // event IDs, last == current + private currentPlaybackId: string; // event ID, broken out from above for ease of use + private recentFullPlays = new Set(); // event IDs + + constructor(private client: MatrixClient, private room: Room) { + this.loadClocks(); + + RoomViewStore.addListener(() => { + if (RoomViewStore.getRoomId() === this.room.roomId) { + // Reset the state of the playbacks before they start mounting and enqueuing updates. + // We reset the entirety of the queue, including order, to ensure the user isn't left + // confused with what order the messages are playing in. + this.currentPlaybackId = null; // this in particular stops autoplay when the room is switched to + this.recentFullPlays = new Set(); + this.playbackIdOrder = []; + } + }); + } + + public static forRoom(roomId: string): PlaybackQueue { + const cli = MatrixClientPeg.get(); + const room = cli.getRoom(roomId); + if (!room) throw new Error("Unknown room"); + if (PlaybackQueue.queues.has(room.roomId)) { + return PlaybackQueue.queues.get(room.roomId); + } + const queue = new PlaybackQueue(cli, room); + PlaybackQueue.queues.set(room.roomId, queue); + return queue; + } + + private persistClocks() { + localStorage.setItem( + `mx_voice_message_clocks_${this.room.roomId}`, + JSON.stringify(Array.from(this.clockStates.entries())), + ); + } + + private loadClocks() { + const val = localStorage.getItem(`mx_voice_message_clocks_${this.room.roomId}`); + if (!!val) { + this.clockStates = new Map(JSON.parse(val)); + } + } + + public unsortedEnqueue(mxEvent: MatrixEvent, playback: Playback) { + // We don't ever detach our listeners: we expect the Playback to clean up for us + this.playbacks.set(mxEvent.getId(), playback); + playback.on(UPDATE_EVENT, (state) => this.onPlaybackStateChange(playback, mxEvent, state)); + playback.clockInfo.liveData.onUpdate((clock) => this.onPlaybackClock(playback, mxEvent, clock)); + } + + private onPlaybackStateChange(playback: Playback, mxEvent: MatrixEvent, newState: PlaybackState) { + // Remember where the user got to in playback + const wasLastPlaying = this.currentPlaybackId === mxEvent.getId(); + if (newState === PlaybackState.Stopped && this.clockStates.has(mxEvent.getId()) && !wasLastPlaying) { + // noinspection JSIgnoredPromiseFromCall + playback.skipTo(this.clockStates.get(mxEvent.getId())); + } else if (newState === PlaybackState.Stopped) { + // Remove the now-useless clock for some space savings + this.clockStates.delete(mxEvent.getId()); + + if (wasLastPlaying) { + this.recentFullPlays.add(this.currentPlaybackId); + const orderClone = arrayFastClone(this.playbackIdOrder); + const last = orderClone.pop(); + if (last === this.currentPlaybackId) { + const next = orderClone.pop(); + if (next) { + const instance = this.playbacks.get(next); + if (!instance) { + console.warn( + "Voice message queue desync: Missing playback for next message: " + + `Current=${this.currentPlaybackId} Last=${last} Next=${next}`, + ); + } else { + this.playbackIdOrder = orderClone; + PlaybackManager.instance.pauseAllExcept(instance); + + // This should cause a Play event, which will re-populate our playback order + // and update our current playback ID. + // noinspection JSIgnoredPromiseFromCall + instance.play(); + } + } else { + // else no explicit next event, so find an event we haven't played that comes next. The live + // timeline is already most recent last, so we can iterate down that. + const timeline = arrayFastClone(this.room.getLiveTimeline().getEvents()); + let scanForVoiceMessage = false; + let nextEv: MatrixEvent; + for (const event of timeline) { + if (event.getId() === mxEvent.getId()) { + scanForVoiceMessage = true; + continue; + } + if (!scanForVoiceMessage) continue; + + // Dev note: This is where we'd break to cause text/non-voice messages to + // interrupt automatic playback. + + const isRightType = isVoiceMessage(event); + const havePlayback = this.playbacks.has(event.getId()); + const isRecentlyCompleted = this.recentFullPlays.has(event.getId()); + if (isRightType && havePlayback && !isRecentlyCompleted) { + nextEv = event; + break; + } + } + if (!nextEv) { + // if we don't have anywhere to go, reset the recent playback queue so the user + // can start a new chain of playbacks. + this.recentFullPlays = new Set(); + this.playbackIdOrder = []; + } else { + this.playbackIdOrder = orderClone; + + const instance = this.playbacks.get(nextEv.getId()); + PlaybackManager.instance.pauseAllExcept(instance); + + // This should cause a Play event, which will re-populate our playback order + // and update our current playback ID. + // noinspection JSIgnoredPromiseFromCall + instance.play(); + } + } + } else { + console.warn( + "Voice message queue desync: Expected playback stop to be last in order. " + + `Current=${this.currentPlaybackId} Last=${last} EventID=${mxEvent.getId()}`, + ); + } + } + } + + if (newState === PlaybackState.Playing) { + const order = this.playbackIdOrder; + if (this.currentPlaybackId !== mxEvent.getId() && !!this.currentPlaybackId) { + if (order.length === 0 || order[order.length - 1] !== this.currentPlaybackId) { + const lastInstance = this.playbacks.get(this.currentPlaybackId); + if ( + lastInstance.currentState === PlaybackState.Playing + || lastInstance.currentState === PlaybackState.Paused + ) { + order.push(this.currentPlaybackId); + } + } + } + + this.currentPlaybackId = mxEvent.getId(); + if (order.length === 0 || order[order.length - 1] !== this.currentPlaybackId) { + order.push(this.currentPlaybackId); + } + } + + // Only persist clock information on pause/stop (end) to avoid overwhelming the storage. + // This should get triggered from normal voice message component unmount due to the playback + // stopping itself for cleanup. + if (newState === PlaybackState.Paused || newState === PlaybackState.Stopped) { + this.persistClocks(); + } + } + + private onPlaybackClock(playback: Playback, mxEvent: MatrixEvent, clocks: number[]) { + if (playback.currentState === PlaybackState.Decoding) return; // ignore pre-ready values + + if (playback.currentState !== PlaybackState.Stopped) { + this.clockStates.set(mxEvent.getId(), clocks[0]); // [0] is the current seek position + } + } +} diff --git a/src/components/views/messages/MAudioBody.tsx b/src/components/views/messages/MAudioBody.tsx index 288ad16d88..1975fe8d42 100644 --- a/src/components/views/messages/MAudioBody.tsx +++ b/src/components/views/messages/MAudioBody.tsx @@ -24,6 +24,8 @@ import { IMediaEventContent } from "../../../customisations/models/IMediaEventCo import MFileBody from "./MFileBody"; import { IBodyProps } from "./IBodyProps"; import { PlaybackManager } from "../../../audio/PlaybackManager"; +import { isVoiceMessage } from "../../../utils/EventUtils"; +import { PlaybackQueue } from "../../../audio/PlaybackQueue"; interface IState { error?: Error; @@ -67,6 +69,10 @@ export default class MAudioBody extends React.PureComponent playback.clockInfo.populatePlaceholdersFrom(this.props.mxEvent); this.setState({ playback }); + if (isVoiceMessage(this.props.mxEvent)) { + PlaybackQueue.forRoom(this.props.mxEvent.getRoomId()).unsortedEnqueue(this.props.mxEvent, playback); + } + // Note: the components later on will handle preparing the Playback class for us. } diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index e8befb90fa..bd573fa474 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -179,7 +179,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent