mirror of
https://github.com/element-hq/element-web
synced 2024-10-24 11:45:50 +03:00
Merge pull request #5955 from matrix-org/travis/voicemessages/timeline
Early rendering for voice messages in the timeline
This commit is contained in:
commit
feac025f6f
8 changed files with 172 additions and 12 deletions
|
@ -161,6 +161,7 @@
|
||||||
@import "./views/messages/_MStickerBody.scss";
|
@import "./views/messages/_MStickerBody.scss";
|
||||||
@import "./views/messages/_MTextBody.scss";
|
@import "./views/messages/_MTextBody.scss";
|
||||||
@import "./views/messages/_MVideoBody.scss";
|
@import "./views/messages/_MVideoBody.scss";
|
||||||
|
@import "./views/messages/_MVoiceMessageBody.scss";
|
||||||
@import "./views/messages/_MessageActionBar.scss";
|
@import "./views/messages/_MessageActionBar.scss";
|
||||||
@import "./views/messages/_MessageTimestamp.scss";
|
@import "./views/messages/_MessageTimestamp.scss";
|
||||||
@import "./views/messages/_MjolnirBody.scss";
|
@import "./views/messages/_MjolnirBody.scss";
|
||||||
|
|
19
res/css/views/messages/_MVoiceMessageBody.scss
Normal file
19
res/css/views/messages/_MVoiceMessageBody.scss
Normal file
|
@ -0,0 +1,19 @@
|
||||||
|
/*
|
||||||
|
Copyright 2021 The Matrix.org Foundation C.I.C.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
.mx_MVoiceMessageBody {
|
||||||
|
display: inline-block; // makes the playback controls magically line up
|
||||||
|
}
|
106
src/components/views/messages/MVoiceMessageBody.tsx
Normal file
106
src/components/views/messages/MVoiceMessageBody.tsx
Normal file
|
@ -0,0 +1,106 @@
|
||||||
|
/*
|
||||||
|
Copyright 2021 The Matrix.org Foundation C.I.C.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import React from "react";
|
||||||
|
import {MatrixEvent} from "matrix-js-sdk/src/models/event";
|
||||||
|
import {replaceableComponent} from "../../../utils/replaceableComponent";
|
||||||
|
import {Playback} from "../../../voice/Playback";
|
||||||
|
import MFileBody from "./MFileBody";
|
||||||
|
import InlineSpinner from '../elements/InlineSpinner';
|
||||||
|
import {_t} from "../../../languageHandler";
|
||||||
|
import {mediaFromContent} from "../../../customisations/Media";
|
||||||
|
import {decryptFile} from "../../../utils/DecryptFile";
|
||||||
|
import RecordingPlayback from "../voice_messages/RecordingPlayback";
|
||||||
|
|
||||||
|
interface IProps {
|
||||||
|
mxEvent: MatrixEvent;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface IState {
|
||||||
|
error?: Error;
|
||||||
|
playback?: Playback;
|
||||||
|
decryptedBlob?: Blob;
|
||||||
|
}
|
||||||
|
|
||||||
|
@replaceableComponent("views.messages.MVoiceMessageBody")
|
||||||
|
export default class MVoiceMessageBody extends React.PureComponent<IProps, IState> {
|
||||||
|
constructor(props: IProps) {
|
||||||
|
super(props);
|
||||||
|
|
||||||
|
this.state = {};
|
||||||
|
}
|
||||||
|
|
||||||
|
public async componentDidMount() {
|
||||||
|
let buffer: ArrayBuffer;
|
||||||
|
const content = this.props.mxEvent.getContent();
|
||||||
|
const media = mediaFromContent(content);
|
||||||
|
if (media.isEncrypted) {
|
||||||
|
try {
|
||||||
|
const blob = await decryptFile(content.file);
|
||||||
|
buffer = await blob.arrayBuffer();
|
||||||
|
this.setState({decryptedBlob: blob});
|
||||||
|
} catch (e) {
|
||||||
|
this.setState({error: e});
|
||||||
|
console.warn("Unable to decrypt voice message", e);
|
||||||
|
return; // stop processing the audio file
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
buffer = await media.downloadSource().then(r => r.blob()).then(r => r.arrayBuffer());
|
||||||
|
} catch (e) {
|
||||||
|
this.setState({error: e});
|
||||||
|
console.warn("Unable to download voice message", e);
|
||||||
|
return; // stop processing the audio file
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const waveform = content?.["org.matrix.msc1767.audio"]?.waveform?.map(p => p / 1024);
|
||||||
|
|
||||||
|
// We should have a buffer to work with now: let's set it up
|
||||||
|
const playback = new Playback(buffer, waveform);
|
||||||
|
this.setState({playback});
|
||||||
|
// Note: the RecordingPlayback component will handle preparing the Playback class for us.
|
||||||
|
}
|
||||||
|
|
||||||
|
public render() {
|
||||||
|
if (this.state.error) {
|
||||||
|
// TODO: @@TR: Verify error state
|
||||||
|
return (
|
||||||
|
<span className="mx_MVoiceMessageBody">
|
||||||
|
<img src={require("../../../../res/img/warning.svg")} width="16" height="16" />
|
||||||
|
{ _t("Error processing voice message") }
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!this.state.playback) {
|
||||||
|
// TODO: @@TR: Verify loading/decrypting state
|
||||||
|
return (
|
||||||
|
<span className="mx_MVoiceMessageBody">
|
||||||
|
<InlineSpinner />
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// At this point we should have a playable state
|
||||||
|
return (
|
||||||
|
<span className="mx_MVoiceMessageBody">
|
||||||
|
<RecordingPlayback playback={this.state.playback} />
|
||||||
|
<MFileBody {...this.props} decryptedBlob={this.state.decryptedBlob} showGenericPlaceholder={false} />
|
||||||
|
</span>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
39
src/components/views/messages/MVoiceOrAudioBody.tsx
Normal file
39
src/components/views/messages/MVoiceOrAudioBody.tsx
Normal file
|
@ -0,0 +1,39 @@
|
||||||
|
/*
|
||||||
|
Copyright 2021 The Matrix.org Foundation C.I.C.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import React from "react";
|
||||||
|
import {MatrixEvent} from "matrix-js-sdk/src/models/event";
|
||||||
|
import MAudioBody from "./MAudioBody";
|
||||||
|
import {replaceableComponent} from "../../../utils/replaceableComponent";
|
||||||
|
import SettingsStore from "../../../settings/SettingsStore";
|
||||||
|
import MVoiceMessageBody from "./MVoiceMessageBody";
|
||||||
|
|
||||||
|
interface IProps {
|
||||||
|
mxEvent: MatrixEvent;
|
||||||
|
}
|
||||||
|
|
||||||
|
@replaceableComponent("views.messages.MVoiceOrAudioBody")
|
||||||
|
export default class MVoiceOrAudioBody extends React.PureComponent<IProps> {
|
||||||
|
public render() {
|
||||||
|
const isVoiceMessage = !!this.props.mxEvent.getContent()['org.matrix.msc2516.voice'];
|
||||||
|
const voiceMessagesEnabled = SettingsStore.getValue("feature_voice_messages");
|
||||||
|
if (isVoiceMessage && voiceMessagesEnabled) {
|
||||||
|
return <MVoiceMessageBody {...this.props} />;
|
||||||
|
} else {
|
||||||
|
return <MAudioBody {...this.props} />;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -72,12 +72,8 @@ export default class MessageEvent extends React.Component {
|
||||||
'm.emote': sdk.getComponent('messages.TextualBody'),
|
'm.emote': sdk.getComponent('messages.TextualBody'),
|
||||||
'm.image': sdk.getComponent('messages.MImageBody'),
|
'm.image': sdk.getComponent('messages.MImageBody'),
|
||||||
'm.file': sdk.getComponent('messages.MFileBody'),
|
'm.file': sdk.getComponent('messages.MFileBody'),
|
||||||
'm.audio': sdk.getComponent('messages.MAudioBody'),
|
'm.audio': sdk.getComponent('messages.MVoiceOrAudioBody'),
|
||||||
'm.video': sdk.getComponent('messages.MVideoBody'),
|
'm.video': sdk.getComponent('messages.MVideoBody'),
|
||||||
|
|
||||||
// TODO: @@ TravisR: Use labs flag determination.
|
|
||||||
// MSC: https://github.com/matrix-org/matrix-doc/pull/2516
|
|
||||||
'org.matrix.msc2516.voice': sdk.getComponent('messages.MAudioBody'),
|
|
||||||
};
|
};
|
||||||
const evTypes = {
|
const evTypes = {
|
||||||
'm.sticker': sdk.getComponent('messages.MStickerBody'),
|
'm.sticker': sdk.getComponent('messages.MStickerBody'),
|
||||||
|
|
|
@ -27,6 +27,7 @@ import LiveRecordingClock from "../voice_messages/LiveRecordingClock";
|
||||||
import {VoiceRecordingStore} from "../../../stores/VoiceRecordingStore";
|
import {VoiceRecordingStore} from "../../../stores/VoiceRecordingStore";
|
||||||
import {UPDATE_EVENT} from "../../../stores/AsyncStore";
|
import {UPDATE_EVENT} from "../../../stores/AsyncStore";
|
||||||
import RecordingPlayback from "../voice_messages/RecordingPlayback";
|
import RecordingPlayback from "../voice_messages/RecordingPlayback";
|
||||||
|
import {MsgType} from "matrix-js-sdk/src/@types/event";
|
||||||
import Modal from "../../../Modal";
|
import Modal from "../../../Modal";
|
||||||
import ErrorDialog from "../dialogs/ErrorDialog";
|
import ErrorDialog from "../dialogs/ErrorDialog";
|
||||||
import CallMediaHandler from "../../../CallMediaHandler";
|
import CallMediaHandler from "../../../CallMediaHandler";
|
||||||
|
@ -67,8 +68,8 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
|
||||||
const mxc = await this.state.recorder.upload();
|
const mxc = await this.state.recorder.upload();
|
||||||
MatrixClientPeg.get().sendMessage(this.props.room.roomId, {
|
MatrixClientPeg.get().sendMessage(this.props.room.roomId, {
|
||||||
"body": "Voice message",
|
"body": "Voice message",
|
||||||
"msgtype": "org.matrix.msc2516.voice",
|
//"msgtype": "org.matrix.msc2516.voice",
|
||||||
//"msgtype": MsgType.Audio,
|
"msgtype": MsgType.Audio,
|
||||||
"url": mxc,
|
"url": mxc,
|
||||||
"info": {
|
"info": {
|
||||||
duration: Math.round(this.state.recorder.durationSeconds * 1000),
|
duration: Math.round(this.state.recorder.durationSeconds * 1000),
|
||||||
|
@ -86,10 +87,6 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
|
||||||
},
|
},
|
||||||
"org.matrix.msc1767.audio": {
|
"org.matrix.msc1767.audio": {
|
||||||
duration: Math.round(this.state.recorder.durationSeconds * 1000),
|
duration: Math.round(this.state.recorder.durationSeconds * 1000),
|
||||||
// TODO: @@ TravisR: Waveform? (MSC1767 decision)
|
|
||||||
},
|
|
||||||
"org.matrix.experimental.msc2516.voice": { // MSC2516+MSC1767 experiment
|
|
||||||
duration: Math.round(this.state.recorder.durationSeconds * 1000),
|
|
||||||
|
|
||||||
// Events can't have floats, so we try to maintain resolution by using 1024
|
// Events can't have floats, so we try to maintain resolution by using 1024
|
||||||
// as a maximum value. The waveform contains values between zero and 1, so this
|
// as a maximum value. The waveform contains values between zero and 1, so this
|
||||||
|
@ -98,6 +95,7 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
|
||||||
// We're expecting about one data point per second of audio.
|
// We're expecting about one data point per second of audio.
|
||||||
waveform: this.state.recorder.getPlayback().waveform.map(v => Math.round(v * 1024)),
|
waveform: this.state.recorder.getPlayback().waveform.map(v => Math.round(v * 1024)),
|
||||||
},
|
},
|
||||||
|
"org.matrix.msc2516.voice": {}, // No content, this is a rendering hint
|
||||||
});
|
});
|
||||||
await this.disposeRecording();
|
await this.disposeRecording();
|
||||||
}
|
}
|
||||||
|
|
|
@ -1852,6 +1852,7 @@
|
||||||
"%(name)s wants to verify": "%(name)s wants to verify",
|
"%(name)s wants to verify": "%(name)s wants to verify",
|
||||||
"You sent a verification request": "You sent a verification request",
|
"You sent a verification request": "You sent a verification request",
|
||||||
"Error decrypting video": "Error decrypting video",
|
"Error decrypting video": "Error decrypting video",
|
||||||
|
"Error processing voice message": "Error processing voice message",
|
||||||
"Show all": "Show all",
|
"Show all": "Show all",
|
||||||
"Reactions": "Reactions",
|
"Reactions": "Reactions",
|
||||||
"<reactors/><reactedWith> reacted with %(content)s</reactedWith>": "<reactors/><reactedWith> reacted with %(content)s</reactedWith>",
|
"<reactors/><reactedWith> reacted with %(content)s</reactedWith>": "<reactors/><reactedWith> reacted with %(content)s</reactedWith>",
|
||||||
|
|
|
@ -50,7 +50,7 @@ export class Playback extends EventEmitter implements IDestroyable {
|
||||||
constructor(private buf: ArrayBuffer, seedWaveform = DEFAULT_WAVEFORM) {
|
constructor(private buf: ArrayBuffer, seedWaveform = DEFAULT_WAVEFORM) {
|
||||||
super();
|
super();
|
||||||
this.context = new AudioContext();
|
this.context = new AudioContext();
|
||||||
this.resampledWaveform = arrayFastResample(seedWaveform, PLAYBACK_WAVEFORM_SAMPLES);
|
this.resampledWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, PLAYBACK_WAVEFORM_SAMPLES);
|
||||||
this.waveformObservable.update(this.resampledWaveform);
|
this.waveformObservable.update(this.resampledWaveform);
|
||||||
this.clock = new PlaybackClock(this.context);
|
this.clock = new PlaybackClock(this.context);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue