mirrors/element-web
1
0
Fork 0
mirror of https://github.com/element-hq/element-web synced 2024-11-23 01:35:49 +03:00
Code Issues Projects Releases Packages Wiki Activity

Hook up a clock and implement proper design

Browse source
This commit is contained in:
Travis Ralston 2021-03-25 17:12:26 -06:00
parent 449e028bbd
commit 1419ac6b69
9 changed files with 222 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
res/css/views/rooms/_VoiceRecordComposerTile.scss
View file

@ -34,3 +34,43 @@ limitations under the License.
background-color: $voice-record-stop-symbol-color;
}
}
.mx_VoiceRecordComposerTile_waveformContainer {
padding: 5px;
padding-right: 4px; // there's 1px from the waveform itself, so account for that
padding-left: 15px; // +10px for the live circle, +5px for regular padding
background-color: $voice-record-waveform-bg-color;
border-radius: 12px;
margin-right: 12px; // isolate from stop button
// Cheat at alignment a bit
display: flex;
align-items: center;
position: relative; // important for the live circle
color: $voice-record-waveform-fg-color;
font-size: $font-14px;
&::before {
// TODO: @@ TravisR: Animate
content: '';
background-color: $voice-record-live-circle-color;
width: 10px;
height: 10px;
position: absolute;
left: 8px;
top: 16px; // vertically center
border-radius: 10px;
}
.mx_Waveform_bar {
background-color: $voice-record-waveform-fg-color;
}
.mx_Clock {
padding-right: 8px; // isolate from waveform
padding-left: 10px; // isolate from live circle
width: 42px; // we're not using a monospace font, so fake it
}
}

20
res/css/views/voice_messages/_Waveform.scss
View file

@ -17,18 +17,24 @@ limitations under the License.
.mx_Waveform {
position: relative;
height: 30px; // tallest bar can only be 30px
top: 1px; // because of our border trick (see below), we're off by 1px of aligntment
display: flex;
align-items: center; // so the bars grow from the middle
overflow: hidden; // this is cheaper than a `max-height: calc(100% - 4px)` in the bar's CSS.
// A bar is meant to be a 2x2 circle when at zero height, and otherwise a 2px wide line
// with rounded caps.
.mx_Waveform_bar {
width: 2px;
margin-left: 1px;
width: 0; // 0px width means we'll end up using the border as our width
border: 1px solid transparent; // transparent means we'll use the background colour
border-radius: 2px; // rounded end caps, based on the border
min-height: 0; // like the width, we'll rely on the border to give us height
max-height: 100%; // this makes the `height: 42%` work on the element
margin-left: 1px; // we want 2px between each bar, so 1px on either side for balance
margin-right: 1px;
background-color: $muted-fg-color;
display: inline-block;
min-height: 2px;
max-height: 100%;
border-radius: 2px; // give them soft endcaps
// background color is handled by the parent components
}
}

3
res/themes/legacy-light/css/_legacy-light.scss
View file

@ -191,6 +191,9 @@ $space-button-outline-color: #E3E8F0;
$voice-record-stop-border-color: #E3E8F0;
$voice-record-stop-symbol-color: $warning-color;
$voice-record-waveform-bg-color: #E3E8F0;
$voice-record-waveform-fg-color: $muted-fg-color;
$voice-record-live-circle-color: $warning-color;
$roomtile-preview-color: #9e9e9e;
$roomtile-default-badge-bg-color: #61708b;

3
res/themes/light/css/_light.scss
View file

@ -182,6 +182,9 @@ $space-button-outline-color: #E3E8F0;
$voice-record-stop-border-color: #E3E8F0;
$voice-record-stop-symbol-color: $warning-color;
$voice-record-waveform-bg-color: #E3E8F0;
$voice-record-waveform-fg-color: $muted-fg-color;
$voice-record-live-circle-color: $warning-color;
$roomtile-preview-color: $secondary-fg-color;
$roomtile-default-badge-bg-color: #61708b;

19
src/components/views/rooms/VoiceRecordComposerTile.tsx
View file

@ -22,6 +22,8 @@ import {Room} from "matrix-js-sdk/src/models/room";
import {MatrixClientPeg} from "../../../MatrixClientPeg";
import classNames from "classnames";
import LiveRecordingWaveform from "../voice_messages/LiveRecordingWaveform";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import LiveRecordingClock from "../voice_messages/LiveRecordingClock";
interface IProps {
room: Room;
@ -32,6 +34,10 @@ interface IState {
recorder?: VoiceRecorder;
}
/**
* Container tile for rendering the voice message recorder in the composer.
*/
@replaceableComponent("views.rooms.VoiceRecordComposerTile")
export default class VoiceRecordComposerTile extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
@ -61,6 +67,15 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
this.setState({recorder});
};
private renderWaveformArea() {
if (!this.state.recorder) return null;
return <div className='mx_VoiceRecordComposerTile_waveformContainer'>
<LiveRecordingClock recorder={this.state.recorder} />
<LiveRecordingWaveform recorder={this.state.recorder} />
</div>;
}
public render() {
const classes = classNames({
'mx_MessageComposer_button': !this.state.recorder,
@ -68,16 +83,14 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
'mx_VoiceRecordComposerTile_stop': !!this.state.recorder,
});
let waveform = null;
let tooltip = _t("Record a voice message");
if (!!this.state.recorder) {
// TODO: @@ TravisR: Change to match behaviour
tooltip = _t("Stop & send recording");
waveform = <LiveRecordingWaveform recorder={this.state.recorder} />;
}
return (<>
{waveform}
{this.renderWaveformArea()}
<AccessibleTooltipButton
className={classes}
onClick={this.onStartStopVoiceMessage}

42
src/components/views/voice_messages/Clock.tsx Normal file
View file

@ -0,0 +1,42 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import {replaceableComponent} from "../../../utils/replaceableComponent";
interface IProps {
seconds: number;
}
interface IState {
}
/**
* Simply converts seconds into minutes and seconds. Note that hours will not be
* displayed, making it possible to see "82:29".
*/
@replaceableComponent("views.voice_messages.Clock")
export default class Clock extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
}
public render() {
const minutes = Math.floor(this.props.seconds / 60).toFixed(0).padStart(2, '0');
const seconds = Math.round(this.props.seconds % 60).toFixed(0).padStart(2, '0'); // hide millis
return <span className='mx_Clock'>{minutes}:{seconds}</span>;
}
}

55
src/components/views/voice_messages/LiveRecordingClock.tsx Normal file
View file

@ -0,0 +1,55 @@
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import {IRecordingUpdate, VoiceRecorder} from "../../../voice/VoiceRecorder";
import {replaceableComponent} from "../../../utils/replaceableComponent";
import Clock from "./Clock";
interface IProps {
recorder: VoiceRecorder;
}
interface IState {
seconds: number;
}
/**
* A clock for a live recording.
*/
@replaceableComponent("views.voice_messages.LiveRecordingClock")
export default class LiveRecordingClock extends React.PureComponent<IProps, IState> {
public constructor(props) {
super(props);
this.state = {seconds: 0};
this.props.recorder.liveData.onUpdate(this.onRecordingUpdate);
}
shouldComponentUpdate(nextProps: Readonly<IProps>, nextState: Readonly<IState>, nextContext: any): boolean {
const currentFloor = Math.floor(this.state.seconds);
const nextFloor = Math.floor(nextState.seconds);
return currentFloor !== nextFloor;
}
private onRecordingUpdate = (update: IRecordingUpdate) => {
this.setState({seconds: update.timeSeconds});
};
public render() {
return <Clock seconds={this.state.seconds} />;
}
}

4
src/components/views/voice_messages/LiveRecordingWaveform.tsx
View file

@ -49,12 +49,12 @@ export default class LiveRecordingWaveform extends React.PureComponent<IProps, I
const bars = arrayFastResample(Array.from(update.waveform), DOWNSAMPLE_TARGET);
this.setState({
// The incoming data is between zero and one, but typically even screaming into a
// microphone won't send you over 0.6, so we "cap" the graph at about 0.4 for a
// microphone won't send you over 0.6, so we "cap" the graph at about 0.35 for a
// point where the average user can still see feedback and be perceived as peaking
// when talking "loudly".
//
// We multiply by 100 because the Waveform component wants values in 0-100 (percentages)
heights: bars.map(b => percentageOf(b, 0, 0.40) * 100),
heights: bars.map(b => percentageOf(b, 0, 0.35) * 100),
});
};

82
src/voice/VoiceRecorder.ts
View file

@ -23,12 +23,10 @@ import {SimpleObservable} from "matrix-widget-api";
const CHANNELS = 1; // stereo isn't important
const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality.
const BITRATE = 24000; // 24kbps is pretty high quality for our use case in opus.
const FREQ_SAMPLE_RATE = 10; // Target rate of frequency data (samples / sec). We don't need this super often.
export interface IRecordingUpdate {
waveform: number[]; // floating points between 0 (low) and 1 (high).
// TODO: @@ TravisR: Generalize this for a timing package?
timeSeconds: number; // float
}
export class VoiceRecorder {
@ -37,11 +35,11 @@ export class VoiceRecorder {
private recorderSource: MediaStreamAudioSourceNode;
private recorderStream: MediaStream;
private recorderFFT: AnalyserNode;
private recorderProcessor: ScriptProcessorNode;
private buffer = new Uint8Array(0);
private mxc: string;
private recording = false;
private observable: SimpleObservable<IRecordingUpdate>;
private freqTimerId: number;
public constructor(private client: MatrixClient) {
}
@ -71,7 +69,20 @@ export class VoiceRecorder {
// it makes the time domain less than helpful.
this.recorderFFT.fftSize = 64;
// We use an audio processor to get accurate timing information.
// The size of the audio buffer largely decides how quickly we push timing/waveform data
// out of this class. Smaller buffers mean we update more frequently as we can't hold as
// many bytes. Larger buffers mean slower updates. For scale, 1024 gives us about 30Hz of
// updates and 2048 gives us about 20Hz. We use 2048 because it updates frequently enough
// to feel realtime (~20fps, which is what humans perceive as "realtime"). Must be a power
// of 2.
this.recorderProcessor = this.recorderContext.createScriptProcessor(2048, CHANNELS, CHANNELS);
// Connect our inputs and outputs
this.recorderSource.connect(this.recorderFFT);
this.recorderSource.connect(this.recorderProcessor);
this.recorderProcessor.connect(this.recorderContext.destination);
this.recorder = new Recorder({
encoderPath, // magic from webpack
encoderSampleRate: SAMPLE_RATE,
@ -117,6 +128,37 @@ export class VoiceRecorder {
return this.mxc;
}
private tryUpdateLiveData = (ev: AudioProcessingEvent) => {
if (!this.recording) return;
// The time domain is the input to the FFT, which means we use an array of the same
// size. The time domain is also known as the audio waveform. We're ignoring the
// output of the FFT here (frequency data) because we're not interested in it.
//
// We use bytes out of the analyser because floats have weird precision problems
// and are slightly more difficult to work with. The bytes are easy to work with,
// which is why we pick them (they're also more precise, but we care less about that).
const data = new Uint8Array(this.recorderFFT.fftSize);
this.recorderFFT.getByteTimeDomainData(data);
// Because we're dealing with a uint array we need to do math a bit differently.
// If we just `Array.from()` the uint array, we end up with 1s and 0s, which aren't
// what we're after. Instead, we have to use a bit of manual looping to correctly end
// up with the right values
const translatedData: number[] = [];
for (let i = 0; i < data.length; i++) {
// All we're doing here is inverting the amplitude and putting the metric somewhere
// between zero and one. Without the inversion, lower values are "louder", which is
// not super helpful.
translatedData.push(1 - (data[i] / 128.0));
}
this.observable.update({
waveform: translatedData,
timeSeconds: ev.playbackTime,
});
};
public async start(): Promise<void> {
if (this.mxc || this.hasRecording) {
throw new Error("Recording already prepared");
@ -129,35 +171,7 @@ export class VoiceRecorder {
}
this.observable = new SimpleObservable<IRecordingUpdate>();
await this.makeRecorder();
this.freqTimerId = setInterval(() => {
if (!this.recording) return;
// The time domain is the input to the FFT, which means we use an array of the same
// size. The time domain is also known as the audio waveform. We're ignoring the
// output of the FFT here (frequency data) because we're not interested in it.
//
// We use bytes out of the analyser because floats have weird precision problems
// and are slightly more difficult to work with. The bytes are easy to work with,
// which is why we pick them (they're also more precise, but we care less about that).
const data = new Uint8Array(this.recorderFFT.fftSize);
this.recorderFFT.getByteTimeDomainData(data);
// Because we're dealing with a uint array we need to do math a bit differently.
// If we just `Array.from()` the uint array, we end up with 1s and 0s, which aren't
// what we're after. Instead, we have to use a bit of manual looping to correctly end
// up with the right values
const translatedData: number[] = [];
for (let i = 0; i < data.length; i++) {
// All we're doing here is inverting the amplitude and putting the metric somewhere
// between zero and one. Without the inversion, lower values are "louder", which is
// not super helpful.
translatedData.push(1 - (data[i] / 128.0));
}
this.observable.update({
waveform: translatedData,
});
}, 1000 / FREQ_SAMPLE_RATE) as any as number; // XXX: Linter doesn't understand timer environment
this.recorderProcessor.addEventListener("audioprocess", this.tryUpdateLiveData);
await this.recorder.start();
this.recording = true;
}
@ -179,8 +193,8 @@ export class VoiceRecorder {
this.recorderStream.getTracks().forEach(t => t.stop());
// Finally do our post-processing and clean up
clearInterval(this.freqTimerId);
this.recording = false;
this.recorderProcessor.removeEventListener("audioprocess", this.tryUpdateLiveData);
await this.recorder.close();
return this.buffer;