Add ringing for matrixRTC (#11870)

* Add ringing for matrixRTC
 - since we are using m.mentions we start with the Notifier
 - an event in the Notifier will result in a IncomingCall toast
 -  incomingCallToast is responsible for ringing (as long as one can see the toast it rings)
 This should make sure visual and audio signal are in sync.

Signed-off-by: Timo K <toger5@hotmail.de>

* use typed CallNotifyContent

Signed-off-by: Timo K <toger5@hotmail.de>

* update tests

Signed-off-by: Timo K <toger5@hotmail.de>

* change to callId

Signed-off-by: Timo K <toger5@hotmail.de>

* fix tests

Signed-off-by: Timo K <toger5@hotmail.de>

* only ring in 1:1 calls
notify in rooms < 15 member

Signed-off-by: Timo K <toger5@hotmail.de>

* call_id fallback

Signed-off-by: Timo K <toger5@hotmail.de>

* Update src/Notifier.ts

Co-authored-by: Robin <robin@robin.town>

* review

Signed-off-by: Timo K <toger5@hotmail.de>

* add tests

Signed-off-by: Timo K <toger5@hotmail.de>

* more tests

Signed-off-by: Timo K <toger5@hotmail.de>

* unused import

Signed-off-by: Timo K <toger5@hotmail.de>

* String -> string

Signed-off-by: Timo K <toger5@hotmail.de>

---------

Signed-off-by: Timo K <toger5@hotmail.de>
Co-authored-by: Robin <robin@robin.town>
This commit is contained in:
Timo 2023-11-21 18:12:08 +01:00 committed by GitHub
parent 7ca0cd13d0
commit a26c2d3c78
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 230 additions and 50 deletions

View file

@ -28,6 +28,7 @@ import {
SyncStateData,
IRoomTimelineData,
M_LOCATION,
EventType,
} from "matrix-js-sdk/src/matrix";
import { logger } from "matrix-js-sdk/src/logger";
import { PermissionChanged as PermissionChangedEvent } from "@matrix-org/analytics-events/types/typescript/PermissionChanged";
@ -54,7 +55,6 @@ import { SdkContextClass } from "./contexts/SDKContext";
import { localNotificationsAreSilenced, createLocalNotificationSettingsIfNeeded } from "./utils/notifications";
import { getIncomingCallToastKey, IncomingCallToast } from "./toasts/IncomingCallToast";
import ToastStore from "./stores/ToastStore";
import { ElementCall } from "./models/Call";
import { VoiceBroadcastChunkEventType, VoiceBroadcastInfoEventType } from "./voice-broadcast";
import { getSenderName } from "./utils/event/getSenderName";
import { stripPlainReply } from "./utils/Reply";
@ -516,13 +516,27 @@ class NotifierClass {
* Some events require special handling such as showing in-app toasts
*/
private performCustomEventHandling(ev: MatrixEvent): void {
if (ElementCall.CALL_EVENT_TYPE.names.includes(ev.getType()) && SettingsStore.getValue("feature_group_calls")) {
if (
EventType.CallNotify === ev.getType() &&
SettingsStore.getValue("feature_group_calls") &&
(ev.getAge() ?? 0) < 10000
) {
const content = ev.getContent();
const roomId = ev.getRoomId();
if (typeof content.call_id !== "string") {
logger.warn("Received malformatted CallNotify event. Did not contain 'call_id' of type 'string'");
return;
}
if (!roomId) {
logger.warn("Could not get roomId for CallNotify event");
return;
}
ToastStore.sharedInstance().addOrReplaceToast({
key: getIncomingCallToastKey(ev.getStateKey()!),
key: getIncomingCallToastKey(content.call_id, roomId),
priority: 100,
component: IncomingCallToast,
bodyClassName: "mx_IncomingCallToast",
props: { callEvent: ev },
props: { notifyEvent: ev },
});
}
}

View file

@ -33,6 +33,8 @@ import { IWidgetApiRequest } from "matrix-widget-api";
import { MatrixRTCSession, MatrixRTCSessionEvent } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
// eslint-disable-next-line no-restricted-imports
import { MatrixRTCSessionManagerEvents } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSessionManager";
// eslint-disable-next-line no-restricted-imports
import { ICallNotifyContent } from "matrix-js-sdk/src/matrixrtc/types";
import type EventEmitter from "events";
import type { ClientWidgetApi } from "matrix-widget-api";
@ -51,6 +53,7 @@ import { getCurrentLanguage } from "../languageHandler";
import { FontWatcher } from "../settings/watchers/FontWatcher";
import { PosthogAnalytics } from "../PosthogAnalytics";
import { UPDATE_EVENT } from "../stores/AsyncStore";
import { getFunctionalMembers } from "../utils/room/getFunctionalMembers";
const TIMEOUT_MS = 16000;
@ -758,10 +761,30 @@ export class ElementCall extends Call {
SettingsStore.getValue("feature_video_rooms") &&
SettingsStore.getValue("feature_element_call_video_rooms") &&
room.isCallRoom();
console.log("Intend is ", isVideoRoom ? "VideoRoom" : "Prompt", " TODO, handle intent appropriately");
ElementCall.createOrGetCallWidget(room.roomId, room.client);
WidgetStore.instance.emit(UPDATE_EVENT, null);
// Send Call notify
const existingRoomCallMembers = MatrixRTCSession.callMembershipsForRoom(room).filter(
// filter all memberships where the application is m.call and the call_id is ""
(m) => m.application === "m.call" && m.callId === "",
);
// We only want to ring in rooms that have less or equal to NOTIFY_MEMBER_LIMIT participants. For really large rooms we don't want to ring.
const NOTIFY_MEMBER_LIMIT = 15;
const memberCount = getFunctionalMembers(room).length;
if (!isVideoRoom && existingRoomCallMembers.length == 0 && memberCount <= NOTIFY_MEMBER_LIMIT) {
// send ringing event
const content: ICallNotifyContent = {
"application": "m.call",
"m.mentions": { user_ids: [], room: true },
"notify_type": memberCount == 2 ? "ring" : "notify",
"call_id": "",
};
await room.client.sendEvent(room.roomId, EventType.CallNotify, content);
}
}
protected async performConnection(

View file

@ -14,8 +14,12 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import React, { useCallback, useEffect } from "react";
import { MatrixEvent, MatrixEventEvent } from "matrix-js-sdk/src/matrix";
import React, { useCallback, useEffect, useMemo } from "react";
import { MatrixEvent } from "matrix-js-sdk/src/matrix";
// eslint-disable-next-line no-restricted-imports
import { MatrixRTCSessionManagerEvents } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSessionManager";
// eslint-disable-next-line no-restricted-imports
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { _t } from "../languageHandler";
import RoomAvatar from "../components/views/avatars/RoomAvatar";
@ -31,14 +35,15 @@ import {
LiveContentType,
} from "../components/views/rooms/LiveContentSummary";
import { useCall, useJoinCallButtonDisabledTooltip } from "../hooks/useCall";
import { useRoomState } from "../hooks/useRoomState";
import { ButtonEvent } from "../components/views/elements/AccessibleButton";
import { useDispatcher } from "../hooks/useDispatcher";
import { ActionPayload } from "../dispatcher/payloads";
import { Call } from "../models/Call";
import { AudioID } from "../LegacyCallHandler";
import { useTypedEventEmitter } from "../hooks/useEventEmitter";
export const getIncomingCallToastKey = (stateKey: string): string => `call_${stateKey}`;
export const getIncomingCallToastKey = (callId: string, roomId: string): string => `call_${callId}_${roomId}`;
const MAX_RING_TIME_MS = 10 * 1000;
interface JoinCallButtonWithCallProps {
onClick: (e: ButtonEvent) => void;
@ -62,36 +67,48 @@ function JoinCallButtonWithCall({ onClick, call }: JoinCallButtonWithCallProps):
}
interface Props {
callEvent: MatrixEvent;
notifyEvent: MatrixEvent;
}
export function IncomingCallToast({ callEvent }: Props): JSX.Element {
const roomId = callEvent.getRoomId()!;
export function IncomingCallToast({ notifyEvent }: Props): JSX.Element {
const roomId = notifyEvent.getRoomId()!;
const room = MatrixClientPeg.safeGet().getRoom(roomId) ?? undefined;
const call = useCall(roomId);
const audio = useMemo(() => document.getElementById(AudioID.Ring) as HTMLMediaElement, []);
// Start ringing if not already.
useEffect(() => {
const isRingToast = (notifyEvent.getContent() as unknown as { notify_type: string })["notify_type"] == "ring";
if (isRingToast && audio.paused) {
audio.play();
}
}, [audio, notifyEvent]);
// Stop ringing on dismiss.
const dismissToast = useCallback((): void => {
ToastStore.sharedInstance().dismissToast(getIncomingCallToastKey(callEvent.getStateKey()!));
}, [callEvent]);
ToastStore.sharedInstance().dismissToast(
getIncomingCallToastKey(notifyEvent.getContent().call_id ?? "", roomId),
);
audio.pause();
}, [audio, notifyEvent, roomId]);
const latestEvent = useRoomState(
room,
useCallback(
(state) => {
return state.getStateEvents(callEvent.getType(), callEvent.getStateKey()!);
},
[callEvent],
),
// Dismiss if session got ended remotely.
const onSessionEnded = useCallback(
(endedSessionRoomId: string, session: MatrixRTCSession): void => {
if (roomId == endedSessionRoomId && session.callId == notifyEvent.getContent().call_id) {
dismissToast();
}
},
[dismissToast, notifyEvent, roomId],
);
// Dismiss on timeout.
useEffect(() => {
if ("m.terminated" in latestEvent.getContent()) {
dismissToast();
}
}, [latestEvent, dismissToast]);
useTypedEventEmitter(latestEvent, MatrixEventEvent.BeforeRedaction, dismissToast);
const timeout = setTimeout(dismissToast, MAX_RING_TIME_MS);
return () => clearTimeout(timeout);
});
// Dismiss on viewing call.
useDispatcher(
defaultDispatcher,
useCallback(
@ -104,21 +121,23 @@ export function IncomingCallToast({ callEvent }: Props): JSX.Element {
),
);
// Dismiss on clicking join.
const onJoinClick = useCallback(
(e: ButtonEvent): void => {
e.stopPropagation();
// The toast will be automatically dismissed by the dispatcher callback above
defaultDispatcher.dispatch<ViewRoomPayload>({
action: Action.ViewRoom,
room_id: room?.roomId,
view_call: true,
metricsTrigger: undefined,
});
dismissToast();
},
[room, dismissToast],
[room],
);
// Dismiss on closing toast.
const onCloseClick = useCallback(
(e: ButtonEvent): void => {
e.stopPropagation();
@ -128,9 +147,17 @@ export function IncomingCallToast({ callEvent }: Props): JSX.Element {
[dismissToast],
);
useTypedEventEmitter(
MatrixClientPeg.safeGet().matrixRTC,
MatrixRTCSessionManagerEvents.SessionEnded,
onSessionEnded,
);
return (
<React.Fragment>
<RoomAvatar room={room ?? undefined} size="24px" />
<div>
<RoomAvatar room={room ?? undefined} size="24px" />
</div>
<div className="mx_IncomingCallToast_content">
<div className="mx_IncomingCallToast_info">
<span className="mx_IncomingCallToast_room">

View file

@ -13,7 +13,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { mocked, MockedObject } from "jest-mock";
import {
ClientEvent,
@ -29,7 +28,6 @@ import {
import { waitFor } from "@testing-library/react";
import BasePlatform from "../src/BasePlatform";
import { ElementCall } from "../src/models/Call";
import Notifier from "../src/Notifier";
import SettingsStore from "../src/settings/SettingsStore";
import ToastStore from "../src/stores/ToastStore";
@ -44,7 +42,7 @@ import {
mockClientMethodsUser,
mockPlatformPeg,
} from "./test-utils";
import { IncomingCallToast } from "../src/toasts/IncomingCallToast";
import { getIncomingCallToastKey, IncomingCallToast } from "../src/toasts/IncomingCallToast";
import { SdkContextClass } from "../src/contexts/SDKContext";
import UserActivity from "../src/UserActivity";
import Modal from "../src/Modal";
@ -389,12 +387,17 @@ describe("Notifier", () => {
jest.resetAllMocks();
});
const callOnEvent = (type?: string) => {
const emitCallNotifyEvent = (type?: string, roomMention = true) => {
const callEvent = mkEvent({
type: type ?? ElementCall.CALL_EVENT_TYPE.name,
type: type ?? EventType.CallNotify,
user: "@alice:foo",
room: roomId,
content: {},
content: {
"application": "m.call",
"m.mentions": { user_ids: [], room: roomMention },
"notify_type": "ring",
"call_id": "abc123",
},
event: true,
});
emitLiveEvent(callEvent);
@ -410,15 +413,15 @@ describe("Notifier", () => {
it("should show toast when group calls are supported", () => {
setGroupCallsEnabled(true);
const callEvent = callOnEvent();
const notifyEvent = emitCallNotifyEvent();
expect(ToastStore.sharedInstance().addOrReplaceToast).toHaveBeenCalledWith(
expect.objectContaining({
key: `call_${callEvent.getStateKey()}`,
key: getIncomingCallToastKey(notifyEvent.getContent().call_id ?? "", roomId),
priority: 100,
component: IncomingCallToast,
bodyClassName: "mx_IncomingCallToast",
props: { callEvent },
props: { notifyEvent },
}),
);
});
@ -426,7 +429,7 @@ describe("Notifier", () => {
it("should not show toast when group calls are not supported", () => {
setGroupCallsEnabled(false);
callOnEvent();
emitCallNotifyEvent();
expect(ToastStore.sharedInstance().addOrReplaceToast).not.toHaveBeenCalled();
});
@ -434,7 +437,7 @@ describe("Notifier", () => {
it("should not show toast when calling with non-group call event", () => {
setGroupCallsEnabled(true);
callOnEvent("event_type");
emitCallNotifyEvent("event_type");
expect(ToastStore.sharedInstance().addOrReplaceToast).not.toHaveBeenCalled();
});

View file

@ -16,6 +16,8 @@ limitations under the License.
import { mocked, Mocked } from "jest-mock";
import { CryptoApi, MatrixClient, Device, Preset, RoomType } from "matrix-js-sdk/src/matrix";
// eslint-disable-next-line no-restricted-imports
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { stubClient, setupAsyncStoreWithClient, mockPlatformPeg, getMockClientWithEventEmitter } from "./test-utils";
import { MatrixClientPeg } from "../src/MatrixClientPeg";
@ -74,6 +76,9 @@ describe("createRoom", () => {
it("sets up Element video rooms correctly", async () => {
const userId = client.getUserId()!;
const createCallSpy = jest.spyOn(ElementCall, "create");
const callMembershipSpy = jest.spyOn(MatrixRTCSession, "callMembershipsForRoom");
callMembershipSpy.mockReturnValue([]);
const roomId = await createRoom(client, { roomType: RoomType.UnstableCall });
const userPower = client.createRoom.mock.calls[0][0].power_level_content_override?.users?.[userId];

View file

@ -17,7 +17,15 @@ limitations under the License.
import EventEmitter from "events";
import { mocked } from "jest-mock";
import { waitFor } from "@testing-library/react";
import { RoomType, Room, RoomEvent, MatrixEvent, RoomStateEvent, PendingEventOrdering } from "matrix-js-sdk/src/matrix";
import {
RoomType,
Room,
RoomEvent,
MatrixEvent,
RoomStateEvent,
PendingEventOrdering,
UNSTABLE_ELEMENT_FUNCTIONAL_USERS,
} from "matrix-js-sdk/src/matrix";
import { Widget } from "matrix-widget-api";
// eslint-disable-next-line no-restricted-imports
import { MatrixRTCSessionManagerEvents } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSessionManager";
@ -982,4 +990,51 @@ describe("ElementCall", () => {
call.off(CallEvent.Destroy, onDestroy);
});
});
describe("create call", () => {
function setFunctionalMembers(members: string[]) {
room.currentState.setStateEvents([
mkEvent({
event: true,
type: UNSTABLE_ELEMENT_FUNCTIONAL_USERS.name,
user: "@user:example.com",
room: room.roomId,
skey: "",
content: { service_members: members },
}),
]);
}
beforeEach(async () => {
setFunctionalMembers(["@user:example.com", "@user2:example.com", "@user4:example.com"]);
});
it("sends notify event on create in a room with more than two members", async () => {
const sendEventSpy = jest.spyOn(room.client, "sendEvent");
await ElementCall.create(room);
expect(sendEventSpy).toHaveBeenCalledWith("!1:example.org", "org.matrix.msc4075.call.notify", {
"application": "m.call",
"call_id": "",
"m.mentions": { room: true, user_ids: [] },
"notify_type": "notify",
});
});
it("sends ring on create in a DM (two participants) room", async () => {
setFunctionalMembers(["@user:example.com", "@user2:example.com"]);
const sendEventSpy = jest.spyOn(room.client, "sendEvent");
await ElementCall.create(room);
expect(sendEventSpy).toHaveBeenCalledWith("!1:example.org", "org.matrix.msc4075.call.notify", {
"application": "m.call",
"call_id": "",
"m.mentions": { room: true, user_ids: [] },
"notify_type": "ring",
});
});
it("don't sent notify event if there are existing room call members", async () => {
jest.spyOn(MatrixRTCSession, "callMembershipsForRoom").mockReturnValue([
{ application: "m.call", callId: "" } as unknown as CallMembership,
]);
const sendEventSpy = jest.spyOn(room.client, "sendEvent");
await ElementCall.create(room);
expect(sendEventSpy).not.toHaveBeenCalled();
});
});
});

View file

@ -19,6 +19,12 @@ import { render, screen, cleanup, fireEvent, waitFor } from "@testing-library/re
import { mocked, Mocked } from "jest-mock";
import { Room, RoomStateEvent, MatrixEvent, MatrixEventEvent, MatrixClient } from "matrix-js-sdk/src/matrix";
import { ClientWidgetApi, Widget } from "matrix-widget-api";
// eslint-disable-next-line no-restricted-imports
import { MatrixRTCSessionManagerEvents } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSessionManager";
// eslint-disable-next-line no-restricted-imports
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
// eslint-disable-next-line no-restricted-imports
import { ICallNotifyContent } from "matrix-js-sdk/src/matrixrtc/types";
import type { RoomMember } from "matrix-js-sdk/src/matrix";
import {
@ -37,6 +43,7 @@ import { WidgetMessagingStore } from "../../src/stores/widgets/WidgetMessagingSt
import DMRoomMap from "../../src/utils/DMRoomMap";
import ToastStore from "../../src/stores/ToastStore";
import { getIncomingCallToastKey, IncomingCallToast } from "../../src/toasts/IncomingCallToast";
import { AudioID } from "../../src/LegacyCallHandler";
describe("IncomingCallEvent", () => {
useMockedCalls();
@ -44,6 +51,7 @@ describe("IncomingCallEvent", () => {
let client: Mocked<MatrixClient>;
let room: Room;
let notifyContent: ICallNotifyContent;
let alice: RoomMember;
let bob: RoomMember;
let call: MockedCall;
@ -59,8 +67,15 @@ describe("IncomingCallEvent", () => {
stubClient();
client = mocked(MatrixClientPeg.safeGet());
room = new Room("!1:example.org", client, "@alice:example.org");
const audio = document.createElement("audio");
audio.id = AudioID.Ring;
document.body.appendChild(audio);
room = new Room("!1:example.org", client, "@alice:example.org");
notifyContent = {
call_id: "",
getRoomId: () => room.roomId,
} as unknown as ICallNotifyContent;
alice = mkRoomMember(room.roomId, "@alice:example.org");
bob = mkRoomMember(room.roomId, "@bob:example.org");
@ -97,7 +112,8 @@ describe("IncomingCallEvent", () => {
});
const renderToast = () => {
render(<IncomingCallToast callEvent={call.event} />);
call.event.getContent = () => notifyContent as any;
render(<IncomingCallToast notifyEvent={call.event} />);
};
it("correctly shows all the information", () => {
@ -115,6 +131,20 @@ describe("IncomingCallEvent", () => {
screen.getByRole("button", { name: "Close" });
});
it("start ringing on ring notify event", () => {
call.event.getContent = () =>
({
...notifyContent,
notify_type: "ring",
} as any);
const playMock = jest.fn();
const audio = { play: playMock, paused: true };
jest.spyOn(document, "getElementById").mockReturnValue(audio as any);
render(<IncomingCallToast notifyEvent={call.event} />);
expect(playMock).toHaveBeenCalled();
});
it("correctly renders toast without a call", () => {
call.destroy();
renderToast();
@ -141,7 +171,9 @@ describe("IncomingCallEvent", () => {
}),
);
await waitFor(() =>
expect(toastStore.dismissToast).toHaveBeenCalledWith(getIncomingCallToastKey(call.event.getStateKey()!)),
expect(toastStore.dismissToast).toHaveBeenCalledWith(
getIncomingCallToastKey(notifyContent.call_id, room.roomId),
),
);
defaultDispatcher.unregister(dispatcherRef);
@ -155,7 +187,9 @@ describe("IncomingCallEvent", () => {
fireEvent.click(screen.getByRole("button", { name: "Close" }));
await waitFor(() =>
expect(toastStore.dismissToast).toHaveBeenCalledWith(getIncomingCallToastKey(call.event.getStateKey()!)),
expect(toastStore.dismissToast).toHaveBeenCalledWith(
getIncomingCallToastKey(notifyContent.call_id, room.roomId),
),
);
defaultDispatcher.unregister(dispatcherRef);
@ -171,7 +205,9 @@ describe("IncomingCallEvent", () => {
});
await waitFor(() =>
expect(toastStore.dismissToast).toHaveBeenCalledWith(getIncomingCallToastKey(call.event.getStateKey()!)),
expect(toastStore.dismissToast).toHaveBeenCalledWith(
getIncomingCallToastKey(notifyContent.call_id, room.roomId),
),
);
});
@ -182,7 +218,24 @@ describe("IncomingCallEvent", () => {
event.emit(MatrixEventEvent.BeforeRedaction, event, {} as unknown as MatrixEvent);
await waitFor(() =>
expect(toastStore.dismissToast).toHaveBeenCalledWith(getIncomingCallToastKey(call.event.getStateKey()!)),
expect(toastStore.dismissToast).toHaveBeenCalledWith(
getIncomingCallToastKey(notifyContent.call_id, room.roomId),
),
);
});
it("closes toast when the matrixRTC session has ended", async () => {
renderToast();
client.matrixRTC.emit(MatrixRTCSessionManagerEvents.SessionEnded, room.roomId, {
callId: notifyContent.call_id,
room: room,
} as unknown as MatrixRTCSession);
await waitFor(() =>
expect(toastStore.dismissToast).toHaveBeenCalledWith(
getIncomingCallToastKey(notifyContent.call_id, room.roomId),
),
);
});
});