/* Copyright 2021 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ import EventEmitter from "events"; import { SimpleObservable } from "matrix-widget-api"; import { logger } from "matrix-js-sdk/src/logger"; import { UPDATE_EVENT } from "../stores/AsyncStore"; import { arrayFastResample, arrayRescale, arraySeed, arraySmoothingResample } from "../utils/arrays"; import { IDestroyable } from "../utils/IDestroyable"; import { PlaybackClock } from "./PlaybackClock"; import { createAudioContext, decodeOgg } from "./compat"; import { clamp } from "../utils/numbers"; export enum PlaybackState { Decoding = "decoding", Stopped = "stopped", // no progress on timeline Paused = "paused", // some progress on timeline Playing = "playing", // active progress through timeline } export interface PlaybackInterface { readonly liveData: SimpleObservable; readonly timeSeconds: number; readonly durationSeconds: number; skipTo(timeSeconds: number): Promise; } export const PLAYBACK_WAVEFORM_SAMPLES = 39; const THUMBNAIL_WAVEFORM_SAMPLES = 100; // arbitrary: [30,120] export const DEFAULT_WAVEFORM = arraySeed(0, PLAYBACK_WAVEFORM_SAMPLES); function makePlaybackWaveform(input: number[]): number[] { // First, convert negative amplitudes to positive so we don't detect zero as "noisy". const noiseWaveform = input.map((v) => Math.abs(v)); // Then, we'll resample the waveform using a smoothing approach so we can keep the same rough shape. // We also rescale the waveform to be 0-1 so we end up with a clamped waveform to rely upon. return arrayRescale(arraySmoothingResample(noiseWaveform, PLAYBACK_WAVEFORM_SAMPLES), 0, 1); } export interface PlaybackInterface { readonly currentState: PlaybackState; readonly liveData: SimpleObservable; readonly timeSeconds: number; readonly durationSeconds: number; skipTo(timeSeconds: number): Promise; } export class Playback extends EventEmitter implements IDestroyable, PlaybackInterface { /** * Stable waveform for representing a thumbnail of the media. Values are * guaranteed to be between zero and one, inclusive. */ public readonly thumbnailWaveform: number[]; private readonly context: AudioContext; private source: AudioBufferSourceNode | MediaElementAudioSourceNode; private state = PlaybackState.Decoding; private audioBuf: AudioBuffer; private element: HTMLAudioElement; private resampledWaveform: number[]; private waveformObservable = new SimpleObservable(); private readonly clock: PlaybackClock; private readonly fileSize: number; /** * Creates a new playback instance from a buffer. * @param {ArrayBuffer} buf The buffer containing the sound sample. * @param {number[]} seedWaveform Optional seed waveform to present until the proper waveform * can be calculated. Contains values between zero and one, inclusive. */ public constructor(private buf: ArrayBuffer, seedWaveform = DEFAULT_WAVEFORM) { super(); // Capture the file size early as reading the buffer will result in a 0-length buffer left behind this.fileSize = this.buf.byteLength; this.context = createAudioContext(); this.resampledWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, PLAYBACK_WAVEFORM_SAMPLES); this.thumbnailWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, THUMBNAIL_WAVEFORM_SAMPLES); this.waveformObservable.update(this.resampledWaveform); this.clock = new PlaybackClock(this.context); } /** * Size of the audio clip in bytes. May be zero if unknown. This is updated * when the playback goes through phase changes. */ public get sizeBytes(): number { return this.fileSize; } /** * Stable waveform for the playback. Values are guaranteed to be between * zero and one, inclusive. */ public get waveform(): number[] { return this.resampledWaveform; } public get waveformData(): SimpleObservable { return this.waveformObservable; } public get clockInfo(): PlaybackClock { return this.clock; } public get liveData(): SimpleObservable { return this.clock.liveData; } public get timeSeconds(): number { return this.clock.timeSeconds; } public get durationSeconds(): number { return this.clock.durationSeconds; } public get currentState(): PlaybackState { return this.state; } public get isPlaying(): boolean { return this.currentState === PlaybackState.Playing; } public emit(event: PlaybackState, ...args: any[]): boolean { this.state = event; super.emit(event, ...args); super.emit(UPDATE_EVENT, event, ...args); return true; // we don't ever care if the event had listeners, so just return "yes" } public destroy(): void { // Dev note: It's critical that we call stop() during cleanup to ensure that downstream callers // are aware of the final clock position before the user triggered an unload. // noinspection JSIgnoredPromiseFromCall - not concerned about being called async here this.stop(); this.removeAllListeners(); this.clock.destroy(); this.waveformObservable.close(); if (this.element) { URL.revokeObjectURL(this.element.src); this.element.remove(); } } public async prepare(): Promise { // don't attempt to decode the media again // AudioContext.decodeAudioData detaches the array buffer `this.buf` // meaning it cannot be re-read if (this.state !== PlaybackState.Decoding) { return; } // The point where we use an audio element is fairly arbitrary, though we don't want // it to be too low. As of writing, voice messages want to show a waveform but audio // messages do not. Using an audio element means we can't show a waveform preview, so // we try to target the difference between a voice message file and large audio file. // Overall, the point of this is to avoid memory-related issues due to storing a massive // audio buffer in memory, as that can balloon to far greater than the input buffer's // byte length. if (this.buf.byteLength > 5 * 1024 * 1024) { // 5mb logger.log("Audio file too large: processing through