From 8ddd14e252dec4f3c87e61492561bb83ae8d6e38 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 22 Mar 2021 20:54:09 -0600 Subject: [PATCH] Early concept for rendering the frequency waveform --- res/css/_components.scss | 1 + .../views/voice_messages/_FrequencyBars.scss | 34 +++++++++++ .../views/rooms/VoiceRecordComposerTile.tsx | 12 ++-- .../views/voice_messages/FrequencyBars.tsx | 58 +++++++++++++++++++ src/utils/arrays.ts | 35 +++++++++++ src/voice/VoiceRecorder.ts | 4 +- 6 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 res/css/views/voice_messages/_FrequencyBars.scss create mode 100644 src/components/views/voice_messages/FrequencyBars.tsx diff --git a/res/css/_components.scss b/res/css/_components.scss index 9c895490b3..33dc6e72cf 100644 --- a/res/css/_components.scss +++ b/res/css/_components.scss @@ -246,6 +246,7 @@ @import "./views/toasts/_AnalyticsToast.scss"; @import "./views/toasts/_NonUrgentEchoFailureToast.scss"; @import "./views/verification/_VerificationShowSas.scss"; +@import "./views/voice_messages/_FrequencyBars.scss"; @import "./views/voip/_CallContainer.scss"; @import "./views/voip/_CallView.scss"; @import "./views/voip/_DialPad.scss"; diff --git a/res/css/views/voice_messages/_FrequencyBars.scss b/res/css/views/voice_messages/_FrequencyBars.scss new file mode 100644 index 0000000000..b38cdfff92 --- /dev/null +++ b/res/css/views/voice_messages/_FrequencyBars.scss @@ -0,0 +1,34 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.mx_FrequencyBars { + position: relative; + height: 30px; // tallest bar can only be 30px + + display: flex; + align-items: center; // so the bars grow from the middle + + .mx_FrequencyBars_bar { + width: 2px; + margin-left: 1px; + margin-right: 1px; + background-color: $muted-fg-color; + display: inline-block; + min-height: 2px; + max-height: 100%; + border-radius: 2px; // give them soft endcaps + } +} diff --git a/src/components/views/rooms/VoiceRecordComposerTile.tsx b/src/components/views/rooms/VoiceRecordComposerTile.tsx index 0d381001a1..c57fc79eeb 100644 --- a/src/components/views/rooms/VoiceRecordComposerTile.tsx +++ b/src/components/views/rooms/VoiceRecordComposerTile.tsx @@ -21,6 +21,7 @@ import {VoiceRecorder} from "../../../voice/VoiceRecorder"; import {Room} from "matrix-js-sdk/src/models/room"; import {MatrixClientPeg} from "../../../MatrixClientPeg"; import classNames from "classnames"; +import FrequencyBars from "../voice_messages/FrequencyBars"; interface IProps { room: Room; @@ -57,10 +58,6 @@ export default class VoiceRecordComposerTile extends React.PureComponent { - // console.log('@@ UPDATE', freq); - // }); this.setState({recorder}); }; @@ -71,18 +68,21 @@ export default class VoiceRecordComposerTile extends React.PureComponent; } - return ( + return (<> + {bars} - ); + ); } } diff --git a/src/components/views/voice_messages/FrequencyBars.tsx b/src/components/views/voice_messages/FrequencyBars.tsx new file mode 100644 index 0000000000..73ea7bc862 --- /dev/null +++ b/src/components/views/voice_messages/FrequencyBars.tsx @@ -0,0 +1,58 @@ +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import React from "react"; +import {IFrequencyPackage, VoiceRecorder} from "../../../voice/VoiceRecorder"; +import {replaceableComponent} from "../../../utils/replaceableComponent"; +import {arrayFastResample, arraySeed} from "../../../utils/arrays"; +import {percentageOf} from "../../../utils/numbers"; + +interface IProps { + recorder: VoiceRecorder +} + +interface IState { + heights: number[]; +} + +const DOWNSAMPLE_TARGET = 35; // number of bars + +@replaceableComponent("views.voice_messages.FrequencyBars") +export default class FrequencyBars extends React.PureComponent { + public constructor(props) { + super(props); + + this.state = {heights: arraySeed(0, DOWNSAMPLE_TARGET)}; + this.props.recorder.frequencyData.onUpdate(this.onFrequencyData); + } + + private onFrequencyData = (freq: IFrequencyPackage) => { + // We're downsampling from about 1024 points to about 35, so this function is fine (see docs/impl) + const bars = arrayFastResample(Array.from(freq.dbBars), DOWNSAMPLE_TARGET); + this.setState({ + // Values are somewhat arbitrary, but help decide what shape the graph should be + heights: bars.map(b => percentageOf(b, -150, -70) * 100), + }); + }; + + public render() { + return
+ {this.state.heights.map((h, i) => { + return ; + })} +
; + } +} diff --git a/src/utils/arrays.ts b/src/utils/arrays.ts index fa5515878f..52308937f7 100644 --- a/src/utils/arrays.ts +++ b/src/utils/arrays.ts @@ -14,6 +14,41 @@ See the License for the specific language governing permissions and limitations under the License. */ +/** + * Quickly resample an array to have less data points. This isn't a perfect representation, + * though this does work best if given a large array to downsample to a much smaller array. + * @param {number[]} input The input array to downsample. + * @param {number} points The number of samples to end up with. + * @returns {number[]} The downsampled array. + */ +export function arrayFastResample(input: number[], points: number): number[] { + // Heavily inpired by matrix-media-repo (used with permission) + // https://github.com/turt2live/matrix-media-repo/blob/abe72c87d2e29/util/util_audio/fastsample.go#L10 + const everyNth = Math.round(input.length / points); + const samples: number[] = []; + for (let i = 0; i < input.length; i += everyNth) { + samples.push(input[i]); + } + while (samples.length < points) { + samples.push(input[input.length - 1]); + } + return samples; +} + +/** + * Creates an array of the given length, seeded with the given value. + * @param {T} val The value to seed the array with. + * @param {number} length The length of the array to create. + * @returns {T[]} The array. + */ +export function arraySeed(val: T, length: number): T[] { + const a: T[] = []; + for (let i = 0; i < length; i++) { + a.push(val); + } + return a; +} + /** * Clones an array as fast as possible, retaining references of the array's values. * @param a The array to clone. Must be defined. diff --git a/src/voice/VoiceRecorder.ts b/src/voice/VoiceRecorder.ts index 06c0d939fc..4bdd0b0af3 100644 --- a/src/voice/VoiceRecorder.ts +++ b/src/voice/VoiceRecorder.ts @@ -23,7 +23,7 @@ import {SimpleObservable} from "matrix-widget-api"; const CHANNELS = 1; // stereo isn't important const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality. const BITRATE = 24000; // 24kbps is pretty high quality for our use case in opus. -const FREQ_SAMPLE_RATE = 4; // Target rate of frequency data (samples / sec). We don't need this super often. +const FREQ_SAMPLE_RATE = 10; // Target rate of frequency data (samples / sec). We don't need this super often. export interface IFrequencyPackage { dbBars: Float32Array; @@ -60,7 +60,7 @@ export class VoiceRecorder { }, }); this.recorderContext = new AudioContext({ - latencyHint: "interactive", + // latencyHint: "interactive", // we don't want a latency hint (this causes data smoothing) sampleRate: SAMPLE_RATE, // once again, the browser will resample for us }); this.recorderSource = this.recorderContext.createMediaStreamSource(this.recorderStream);