From 55f77b04ae118f2fad499c9c8ff87be416cbc37d Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 9 Oct 2020 18:56:07 +0100 Subject: [PATCH] Rewrite call state machine * Remove the two separate enumerations of call state: now everything uses the js-sdk version of call state. Stop adding a separate 'call_state' field onto the call object(!) * Better reflection of the actual state of the call in the call bar, so when it's connecting, it says connecting, and only says 'active call' when the call is actually active. * More typey goodness --- src/@types/global.d.ts | 1 + src/CallHandler.tsx | 176 +++++++++--------- src/components/structures/RoomStatusBar.js | 45 +++-- src/components/structures/RoomView.tsx | 42 ++--- src/components/views/voip/CallPreview.tsx | 7 +- src/components/views/voip/CallView.tsx | 5 +- src/components/views/voip/IncomingCallBox.tsx | 3 +- src/i18n/strings/en_EN.json | 4 + 8 files changed, 153 insertions(+), 130 deletions(-) diff --git a/src/@types/global.d.ts b/src/@types/global.d.ts index 91b91de90d..93be0fafc0 100644 --- a/src/@types/global.d.ts +++ b/src/@types/global.d.ts @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +import "matrix-js-sdk/src/@types/global"; // load matrix-js-sdk's type extensions first import * as ModernizrStatic from "modernizr"; import ContentMessages from "../ContentMessages"; import { IMatrixClientPeg } from "../MatrixClientPeg"; diff --git a/src/CallHandler.tsx b/src/CallHandler.tsx index 5b368016b6..4a71934b7f 100644 --- a/src/CallHandler.tsx +++ b/src/CallHandler.tsx @@ -77,13 +77,18 @@ import ErrorDialog from "./components/views/dialogs/ErrorDialog"; import WidgetStore from "./stores/WidgetStore"; import { WidgetMessagingStore } from "./stores/widgets/WidgetMessagingStore"; import { ElementWidgetActions } from "./stores/widgets/ElementWidgetActions"; +import { MatrixCall, CallErrorCode, CallState, CallType } from "matrix-js-sdk/lib/webrtc/call"; -// until we ts-ify the js-sdk voip code -type Call = any; +enum AudioId { + Ring = 'ringAudio', + Ringback = 'ringbackAudio', + CallEnd = 'callendAudio', + Busy = 'busyAudio', +} export default class CallHandler { - private calls = new Map(); - private audioPromises = new Map>(); + private calls = new Map(); + private audioPromises = new Map>(); static sharedInstance() { if (!window.mxCallHandler) { @@ -108,7 +113,7 @@ export default class CallHandler { } } - getCallForRoom(roomId: string): Call { + getCallForRoom(roomId: string): MatrixCall { return this.calls.get(roomId) || null; } @@ -121,7 +126,7 @@ export default class CallHandler { return null; } - play(audioId: string) { + play(audioId: AudioId) { // TODO: Attach an invisible element for this instead // which listens? const audio = document.getElementById(audioId) as HTMLMediaElement; @@ -150,7 +155,7 @@ export default class CallHandler { } } - pause(audioId: string) { + pause(audioId: AudioId) { // TODO: Attach an invisible element for this instead // which listens? const audio = document.getElementById(audioId) as HTMLMediaElement; @@ -164,7 +169,7 @@ export default class CallHandler { } } - private setCallListeners(call: Call) { + private setCallListeners(call: MatrixCall) { call.on("error", (err) => { console.error("Call error:", err); if ( @@ -185,69 +190,57 @@ export default class CallHandler { }); // map web rtc states to dummy UI state // ringing|ringback|connected|ended|busy|stop_ringback|stop_ringing - call.on("state", (newState, oldState) => { - if (newState === "ringing") { - this.setCallState(call, call.roomId, "ringing"); - this.pause("ringbackAudio"); - } else if (newState === "invite_sent") { - this.setCallState(call, call.roomId, "ringback"); - this.play("ringbackAudio"); - } else if (newState === "ended" && oldState === "connected") { - this.removeCallForRoom(call.roomId); - this.pause("ringbackAudio"); - this.play("callendAudio"); - } else if (newState === "ended" && oldState === "invite_sent" && - (call.hangupParty === "remote" || - (call.hangupParty === "local" && call.hangupReason === "invite_timeout") + call.on("state", (newState: CallState, oldState: CallState) => { + this.setCallState(call, newState); + + switch (oldState) { + case CallState.Ringing: + this.pause(AudioId.Ring); + break; + case CallState.InviteSent: + this.pause(AudioId.Ringback); + break; + } + + switch (newState) { + case CallState.Ringing: + this.play(AudioId.Ring); + break; + case CallState.InviteSent: + this.play(AudioId.Ringback); + break; + case CallState.Ended: + this.removeCallForRoom(call.roomId); + if (oldState === CallState.InviteSent && ( + call.hangupParty === "remote" || + (call.hangupParty === "local" && call.hangupReason === "invite_timeout") )) { - this.setCallState(call, call.roomId, "busy"); - this.pause("ringbackAudio"); - this.play("busyAudio"); - Modal.createTrackedDialog('Call Handler', 'Call Timeout', ErrorDialog, { - title: _t('Call Timeout'), - description: _t('The remote side failed to pick up') + '.', - }); - } else if (oldState === "invite_sent") { - this.setCallState(call, call.roomId, "stop_ringback"); - this.pause("ringbackAudio"); - } else if (oldState === "ringing") { - this.setCallState(call, call.roomId, "stop_ringing"); - this.pause("ringbackAudio"); - } else if (newState === "connected") { - this.setCallState(call, call.roomId, "connected"); - this.pause("ringbackAudio"); + this.play(AudioId.Busy); + Modal.createTrackedDialog('Call Handler', 'Call Timeout', ErrorDialog, { + title: _t('Call Timeout'), + description: _t('The remote side failed to pick up') + '.', + }); + } else { + this.play(AudioId.CallEnd); + } } }); } - private setCallState(call: Call, roomId: string, status: string) { + private setCallState(call: MatrixCall, status: CallState) { console.log( - `Call state in ${roomId} changed to ${status} (${call ? call.call_state : "-"})`, + `Call state in ${call.roomId} changed to ${status}`, ); - if (call) { - this.calls.set(roomId, call); - } else { - this.calls.delete(roomId); - } - if (status === "ringing") { - this.play("ringAudio"); - } else if (call && call.call_state === "ringing") { - this.pause("ringAudio"); - } - - if (call) { - call.call_state = status; - } dis.dispatch({ action: 'call_state', - room_id: roomId, + room_id: call.roomId, state: status, }); } private removeCallForRoom(roomId: string) { - this.setCallState(null, roomId, null); + this.calls.delete(roomId); } private showICEFallbackPrompt() { @@ -279,36 +272,36 @@ export default class CallHandler { }, null, true); } - private onAction = (payload: ActionPayload) => { - const placeCall = (newCall) => { - this.setCallListeners(newCall); - if (payload.type === 'voice') { - newCall.placeVoiceCall(); - } else if (payload.type === 'video') { - newCall.placeVideoCall( - payload.remote_element, - payload.local_element, - ); - } else if (payload.type === 'screensharing') { - const screenCapErrorString = PlatformPeg.get().screenCaptureErrorString(); - if (screenCapErrorString) { - this.removeCallForRoom(newCall.roomId); - console.log("Can't capture screen: " + screenCapErrorString); - Modal.createTrackedDialog('Call Handler', 'Unable to capture screen', ErrorDialog, { - title: _t('Unable to capture screen'), - description: screenCapErrorString, - }); - return; - } - newCall.placeScreenSharingCall( - payload.remote_element, - payload.local_element, - ); - } else { - console.error("Unknown conf call type: %s", payload.type); - } - } + private placeCall(roomId: string, type: CallType, localElement: HTMLVideoElement, remoteElement: HTMLVideoElement) { + const call = Matrix.createNewMatrixCall(MatrixClientPeg.get(), roomId); + this.calls.set(roomId, call); + this.setCallListeners(call); + if (type === 'voice') { + call.placeVoiceCall(); + } else if (type === 'video') { + call.placeVideoCall( + remoteElement, + localElement, + ); + } else if (type === 'screensharing') { + const screenCapErrorString = PlatformPeg.get().screenCaptureErrorString(); + if (screenCapErrorString) { + this.removeCallForRoom(roomId); + console.log("Can't capture screen: " + screenCapErrorString); + Modal.createTrackedDialog('Call Handler', 'Unable to capture screen', ErrorDialog, { + title: _t('Unable to capture screen'), + description: screenCapErrorString, + }); + return; + } + call.placeScreenSharingCall(remoteElement, localElement); + } else { + console.error("Unknown conf call type: %s", type); + } + } + + private onAction = (payload: ActionPayload) => { switch (payload.action) { case 'place_call': { @@ -343,8 +336,8 @@ export default class CallHandler { return; } else if (members.length === 2) { console.info("Place %s call in %s", payload.type, payload.room_id); - const call = Matrix.createNewMatrixCall(MatrixClientPeg.get(), payload.room_id); - placeCall(call); + + this.placeCall(payload.room_id, payload.type, payload.local_element, payload.remote_element); } else { // > 2 dis.dispatch({ action: "place_conference_call", @@ -383,24 +376,23 @@ export default class CallHandler { return; } - const call = payload.call; + const call = payload.call as MatrixCall; + this.calls.set(call.roomId, call) this.setCallListeners(call); - this.setCallState(call, call.roomId, "ringing"); } break; case 'hangup': if (!this.calls.get(payload.room_id)) { return; // no call to hangup } - this.calls.get(payload.room_id).hangup(); + this.calls.get(payload.room_id).hangup(CallErrorCode.UserHangup, false) this.removeCallForRoom(payload.room_id); break; case 'answer': - if (!this.calls.get(payload.room_id)) { + if (!this.calls.has(payload.room_id)) { return; // no call to answer } this.calls.get(payload.room_id).answer(); - this.setCallState(this.calls.get(payload.room_id), payload.room_id, "connected"); dis.dispatch({ action: "view_room", room_id: payload.room_id, diff --git a/src/components/structures/RoomStatusBar.js b/src/components/structures/RoomStatusBar.js index cdaa0bb7f9..453663a261 100644 --- a/src/components/structures/RoomStatusBar.js +++ b/src/components/structures/RoomStatusBar.js @@ -1,7 +1,5 @@ /* -Copyright 2015, 2016 OpenMarket Ltd -Copyright 2017, 2018 New Vector Ltd -Copyright 2019 The Matrix.org Foundation C.I.C. +Copyright 2015-2020 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -46,10 +44,12 @@ export default class RoomStatusBar extends React.Component { // Used to suggest to the user to invite someone sentMessageAndIsAlone: PropTypes.bool, - // true if there is an active call in this room (means we show - // the 'Active Call' text in the status bar if there is nothing - // more interesting) - hasActiveCall: PropTypes.bool, + // The active call in the room, if any (means we show the call bar + // along with the status of the call) + callState: PropTypes.string, + + // The type of the call in progress, or null if no call is in progress + callType: PropTypes.string, // true if the room is being peeked at. This affects components that shouldn't // logically be shown when peeking, such as a prompt to invite people to a room. @@ -121,6 +121,10 @@ export default class RoomStatusBar extends React.Component { }); }; + _showCallBar() { + return this.props.callState !== 'ended' && this.props.callState !== 'ringing'; + } + _onResendAllClick = () => { Resend.resendUnsentEvents(this.props.room); dis.fire(Action.FocusComposer); @@ -153,7 +157,7 @@ export default class RoomStatusBar extends React.Component { // indicate other sizes. _getSize() { if (this._shouldShowConnectionError() || - this.props.hasActiveCall || + this._showCallBar() || this.props.sentMessageAndIsAlone ) { return STATUS_BAR_EXPANDED; @@ -165,7 +169,7 @@ export default class RoomStatusBar extends React.Component { // return suitable content for the image on the left of the status bar. _getIndicator() { - if (this.props.hasActiveCall) { + if (this._showCallBar()) { const TintableSvg = sdk.getComponent("elements.TintableSvg"); return ( @@ -269,6 +273,25 @@ export default class RoomStatusBar extends React.Component { ; } + _getCallStatusText() { + switch (this.props.callState) { + case 'create_offer': + case 'invite_sent': + return _t('Calling...'); + case 'connecting': + case 'create_answer': + return _t('Call connecting...'); + case 'connected': + return _t('Active call'); + case 'wait_local_media': + if (this.props.callType === 'video') { + return _t('Starting camera...'); + } else { + return _t('Starting microphone...'); + } + } + } + // return suitable content for the main (text) part of the status bar. _getContent() { if (this._shouldShowConnectionError()) { @@ -291,10 +314,10 @@ export default class RoomStatusBar extends React.Component { return this._getUnsentMessageContent(); } - if (this.props.hasActiveCall) { + if (this._showCallBar()) { return (
- { _t('Active call') } + { this._getCallStatusText() }
); } diff --git a/src/components/structures/RoomView.tsx b/src/components/structures/RoomView.tsx index fcb2d274c1..2a6d0b5de8 100644 --- a/src/components/structures/RoomView.tsx +++ b/src/components/structures/RoomView.tsx @@ -71,6 +71,7 @@ import RoomHeader from "../views/rooms/RoomHeader"; import TintableSvg from "../views/elements/TintableSvg"; import {XOR} from "../../@types/common"; import { IThreepidInvite } from "../../stores/ThreepidInviteStore"; +import { CallState, MatrixCall } from "matrix-js-sdk/lib/webrtc/call"; const DEBUG = false; let debuglog = function(msg: string) {}; @@ -141,7 +142,7 @@ export interface IState { }>; searchHighlights?: string[]; searchInProgress?: boolean; - callState?: string; + callState?: CallState; guestsCanJoin: boolean; canPeek: boolean; showApps: boolean; @@ -479,7 +480,7 @@ export default class RoomView extends React.Component { componentDidMount() { const call = this.getCallForRoom(); - const callState = call ? call.call_state : "ended"; + const callState = call ? call.state : null; this.setState({ callState: callState, }); @@ -712,14 +713,9 @@ export default class RoomView extends React.Component { } const call = this.getCallForRoom(); - let callState = "ended"; - - if (call) { - callState = call.call_state; - } this.setState({ - callState: callState, + callState: call ? call.state : null, }); break; } @@ -1605,7 +1601,7 @@ export default class RoomView extends React.Component { /** * get any current call for this room */ - private getCallForRoom() { + private getCallForRoom(): MatrixCall { if (!this.state.room) { return null; } @@ -1742,10 +1738,12 @@ export default class RoomView extends React.Component { // We have successfully loaded this room, and are not previewing. // Display the "normal" room view. - const call = this.getCallForRoom(); - let inCall = false; - if (call && (this.state.callState !== 'ended' && this.state.callState !== 'ringing')) { - inCall = true; + let activeCall = null; + { + const call = this.getCallForRoom(); + if (call && (this.state.callState !== 'ended' && this.state.callState !== 'ringing')) { + activeCall = call; + } } const scrollheaderClasses = classNames({ @@ -1764,7 +1762,8 @@ export default class RoomView extends React.Component { statusBar = { }; } - if (inCall) { + if (activeCall) { let zoomButton; let videoMuteButton; - if (call.type === "video") { + if (activeCall.type === "video") { zoomButton = (
{ videoMuteButton =
@@ -1920,10 +1920,10 @@ export default class RoomView extends React.Component { const voiceMuteButton =
@@ -2041,7 +2041,7 @@ export default class RoomView extends React.Component { }); const mainClasses = classNames("mx_RoomView", { - mx_RoomView_inCall: inCall, + mx_RoomView_inCall: Boolean(activeCall), }); return ( diff --git a/src/components/views/voip/CallPreview.tsx b/src/components/views/voip/CallPreview.tsx index 9acbece8b3..ca2b510f20 100644 --- a/src/components/views/voip/CallPreview.tsx +++ b/src/components/views/voip/CallPreview.tsx @@ -24,13 +24,14 @@ import dis from '../../../dispatcher/dispatcher'; import { ActionPayload } from '../../../dispatcher/payloads'; import PersistentApp from "../elements/PersistentApp"; import SettingsStore from "../../../settings/SettingsStore"; +import { CallState, MatrixCall } from 'matrix-js-sdk/lib/webrtc/call'; interface IProps { } interface IState { roomId: string; - activeCall: any; + activeCall: MatrixCall; } export default class CallPreview extends React.Component { @@ -84,7 +85,7 @@ export default class CallPreview extends React.Component { if (call) { dis.dispatch({ action: 'view_room', - room_id: call.groupRoomId || call.roomId, + room_id: call.roomId, }); } }; @@ -93,7 +94,7 @@ export default class CallPreview extends React.Component { const callForRoom = CallHandler.sharedInstance().getCallForRoom(this.state.roomId); const showCall = ( this.state.activeCall && - this.state.activeCall.call_state === 'connected' && + this.state.activeCall.state === CallState.Connected && !callForRoom ); diff --git a/src/components/views/voip/CallView.tsx b/src/components/views/voip/CallView.tsx index 2ab291ae86..3e1833a903 100644 --- a/src/components/views/voip/CallView.tsx +++ b/src/components/views/voip/CallView.tsx @@ -25,6 +25,7 @@ import AccessibleButton from '../elements/AccessibleButton'; import VideoView from "./VideoView"; import RoomAvatar from "../avatars/RoomAvatar"; import PulsedAvatar from '../avatars/PulsedAvatar'; +import { CallState, MatrixCall } from 'matrix-js-sdk/lib/webrtc/call'; interface IProps { // js-sdk room object. If set, we will only show calls for the given @@ -87,7 +88,7 @@ export default class CallView extends React.Component { }; private showCall() { - let call; + let call: MatrixCall; if (this.props.room) { const roomId = this.props.room.roomId; @@ -120,7 +121,7 @@ export default class CallView extends React.Component { call.setRemoteAudioElement(this.getVideoView().getRemoteAudioElement()); } } - if (call && call.type === "video" && call.call_state !== "ended" && call.call_state !== "ringing") { + if (call && call.type === "video" && call.state !== CallState.Ended && call.state !== CallState.Ringing) { this.getVideoView().getLocalVideoElement().style.display = "block"; this.getVideoView().getRemoteVideoElement().style.display = "block"; } else { diff --git a/src/components/views/voip/IncomingCallBox.tsx b/src/components/views/voip/IncomingCallBox.tsx index 8e5d0f9e4a..560a034f47 100644 --- a/src/components/views/voip/IncomingCallBox.tsx +++ b/src/components/views/voip/IncomingCallBox.tsx @@ -25,6 +25,7 @@ import CallHandler from '../../../CallHandler'; import PulsedAvatar from '../avatars/PulsedAvatar'; import RoomAvatar from '../avatars/RoomAvatar'; import FormButton from '../elements/FormButton'; +import { CallState } from 'matrix-js-sdk/lib/webrtc/call'; interface IProps { } @@ -53,7 +54,7 @@ export default class IncomingCallBox extends React.Component { switch (payload.action) { case 'call_state': { const call = CallHandler.sharedInstance().getCallForRoom(payload.room_id); - if (call && call.call_state === 'ringing') { + if (call && call.state === CallState.Ringing) { this.setState({ incomingCall: call, }); diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index d3b942e6fa..eb8f9100ec 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -2095,6 +2095,10 @@ "%(count)s of your messages have not been sent.|one": "Your message was not sent.", "%(count)s Resend all or cancel all now. You can also select individual messages to resend or cancel.|other": "Resend all or cancel all now. You can also select individual messages to resend or cancel.", "%(count)s Resend all or cancel all now. You can also select individual messages to resend or cancel.|one": "Resend message or cancel message now.", + "Calling...": "Calling...", + "Call connecting...": "Call connecting...", + "Starting camera...": "Starting camera...", + "Starting microphone...": "Starting microphone...", "Connectivity to the server has been lost.": "Connectivity to the server has been lost.", "Sent messages will be stored until your connection has returned.": "Sent messages will be stored until your connection has returned.", "There's no one else here! Would you like to invite others or stop warning about the empty room?": "There's no one else here! Would you like to invite others or stop warning about the empty room?",