Advanced audio processing settings (#8759)

Co-authored-by: Šimon Brandner <simon.bra.ag@gmail.com>
Fixes https://github.com/vector-im/element-web/issues/6278
Fixes undefined

src/MediaDeviceHandler.ts

@@ -88,6 +88,16 @@ export default class MediaDeviceHandler extends EventEmitter {
 
         await MatrixClientPeg.get().getMediaHandler().setAudioInput(audioDeviceId);
         await MatrixClientPeg.get().getMediaHandler().setVideoInput(videoDeviceId);
+
+        await MediaDeviceHandler.updateAudioSettings();
+    }
+
+    private static async updateAudioSettings(): Promise<void> {
+        await MatrixClientPeg.get().getMediaHandler().setAudioSettings({
+            autoGainControl: MediaDeviceHandler.getAudioAutoGainControl(),
+            echoCancellation: MediaDeviceHandler.getAudioEchoCancellation(),
+            noiseSuppression: MediaDeviceHandler.getAudioNoiseSuppression(),
+        });
     }
 
     public setAudioOutput(deviceId: string): void {
@@ -123,6 +133,21 @@ export default class MediaDeviceHandler extends EventEmitter {
         }
     }
 
+    public static async setAudioAutoGainControl(value: boolean): Promise<void> {
+        await SettingsStore.setValue("webrtc_audio_autoGainControl", null, SettingLevel.DEVICE, value);
+        await MediaDeviceHandler.updateAudioSettings();
+    }
+
+    public static async setAudioEchoCancellation(value: boolean): Promise<void> {
+        await SettingsStore.setValue("webrtc_audio_echoCancellation", null, SettingLevel.DEVICE, value);
+        await MediaDeviceHandler.updateAudioSettings();
+    }
+
+    public static async setAudioNoiseSuppression(value: boolean): Promise<void> {
+        await SettingsStore.setValue("webrtc_audio_noiseSuppression", null, SettingLevel.DEVICE, value);
+        await MediaDeviceHandler.updateAudioSettings();
+    }
+
     public static getAudioOutput(): string {
         return SettingsStore.getValueAt(SettingLevel.DEVICE, "webrtc_audiooutput");
     }
@@ -135,6 +160,18 @@ export default class MediaDeviceHandler extends EventEmitter {
         return SettingsStore.getValueAt(SettingLevel.DEVICE, "webrtc_videoinput");
     }
 
+    public static getAudioAutoGainControl(): boolean {
+        return SettingsStore.getValue("webrtc_audio_autoGainControl");
+    }
+
+    public static getAudioEchoCancellation(): boolean {
+        return SettingsStore.getValue("webrtc_audio_echoCancellation");
+    }
+
+    public static getAudioNoiseSuppression(): boolean {
+        return SettingsStore.getValue("webrtc_audio_noiseSuppression");
+    }
+
     /**
      * Returns the current set deviceId for a device kind
      * @param {MediaDeviceKindEnum} kind of the device that will be returned

src/components/views/settings/tabs/user/VoiceUserSettingsTab.tsx

@@ -27,6 +27,7 @@ import { MatrixClientPeg } from "../../../../../MatrixClientPeg";
 import Modal from "../../../../../Modal";
 import { SettingLevel } from "../../../../../settings/SettingLevel";
 import SettingsFlag from '../../../elements/SettingsFlag';
+import LabelledToggleSwitch from "../../../elements/LabelledToggleSwitch";
 import ErrorDialog from '../../../dialogs/ErrorDialog';
 
 const getDefaultDevice = (devices: Array<Partial<MediaDeviceInfo>>) => {
@@ -41,8 +42,14 @@ const getDefaultDevice = (devices: Array<Partial<MediaDeviceInfo>>) => {
     }
 };
 
-interface IState extends Record<MediaDeviceKindEnum, string> {
+interface IState {
     mediaDevices: IMediaDevices;
+    [MediaDeviceKindEnum.AudioOutput]: string;
+    [MediaDeviceKindEnum.AudioInput]: string;
+    [MediaDeviceKindEnum.VideoInput]: string;
+    audioAutoGainControl: boolean;
+    audioEchoCancellation: boolean;
+    audioNoiseSuppression: boolean;
 }
 
 export default class VoiceUserSettingsTab extends React.Component<{}, IState> {
@@ -54,6 +61,9 @@ export default class VoiceUserSettingsTab extends React.Component<{}, IState> {
             [MediaDeviceKindEnum.AudioOutput]: null,
             [MediaDeviceKindEnum.AudioInput]: null,
             [MediaDeviceKindEnum.VideoInput]: null,
+            audioAutoGainControl: MediaDeviceHandler.getAudioAutoGainControl(),
+            audioEchoCancellation: MediaDeviceHandler.getAudioEchoCancellation(),
+            audioNoiseSuppression: MediaDeviceHandler.getAudioNoiseSuppression(),
         };
     }
 
@@ -183,22 +193,63 @@ export default class VoiceUserSettingsTab extends React.Component<{}, IState> {
         return (
             <div className="mx_SettingsTab mx_VoiceUserSettingsTab">
                 <div className="mx_SettingsTab_heading">{ _t("Voice & Video") }</div>
+                { requestButton }
                 <div className="mx_SettingsTab_section">
-                    { requestButton }
+                    <span className="mx_SettingsTab_subheading">{ _t("Voice settings") }</span>
                     { speakerDropdown }
                     { microphoneDropdown }
+                    <LabelledToggleSwitch
+                        value={this.state.audioAutoGainControl}
+                        onChange={async (v) => {
+                            await MediaDeviceHandler.setAudioAutoGainControl(v);
+                            this.setState({ audioAutoGainControl: MediaDeviceHandler.getAudioAutoGainControl() });
+                        }}
+                        label={_t("Automatically adjust the microphone volume")}
+                        data-testid='voice-auto-gain'
+                    />
+                </div>
+                <div className="mx_SettingsTab_section">
+                    <span className="mx_SettingsTab_subheading">{ _t("Video settings") }</span>
                     { webcamDropdown }
                     <SettingsFlag name='VideoView.flipVideoHorizontally' level={SettingLevel.ACCOUNT} />
-                    <SettingsFlag
+                </div>
-                        name='webRtcAllowPeerToPeer'
+
-                        level={SettingLevel.DEVICE}
+                <div className="mx_SettingsTab_heading">{ _t("Advanced") }</div>
-                        onChange={this.changeWebRtcMethod}
+                <div className="mx_SettingsTab_section">
-                    />
+                    <span className="mx_SettingsTab_subheading">{ _t("Voice processing") }</span>
-                    <SettingsFlag
+                    <div className="mx_SettingsTab_section">
-                        name='fallbackICEServerAllowed'
+                        <LabelledToggleSwitch
-                        level={SettingLevel.DEVICE}
+                            value={this.state.audioNoiseSuppression}
-                        onChange={this.changeFallbackICEServerAllowed}
+                            onChange={async (v) => {
-                    />
+                                await MediaDeviceHandler.setAudioNoiseSuppression(v);
+                                this.setState({ audioNoiseSuppression: MediaDeviceHandler.getAudioNoiseSuppression() });
+                            }}
+                            label={_t("Noise suppression")}
+                            data-testid='voice-noise-suppression'
+                        />
+                        <LabelledToggleSwitch
+                            value={this.state.audioEchoCancellation}
+                            onChange={async (v) => {
+                                await MediaDeviceHandler.setAudioEchoCancellation(v);
+                                this.setState({ audioEchoCancellation: MediaDeviceHandler.getAudioEchoCancellation() });
+                            }}
+                            label={_t("Echo cancellation")}
+                            data-testid='voice-echo-cancellation'
+                        />
+                    </div>
+                    <div className="mx_SettingsTab_section">
+                        <span className="mx_SettingsTab_subheading">{ _t("Connection") }</span>
+                        <SettingsFlag
+                            name='webRtcAllowPeerToPeer'
+                            level={SettingLevel.DEVICE}
+                            onChange={this.changeWebRtcMethod}
+                        />
+                        <SettingsFlag
+                            name='fallbackICEServerAllowed'
+                            level={SettingLevel.DEVICE}
+                            onChange={this.changeFallbackICEServerAllowed}
+                        />
+                    </div>
                 </div>
             </div>
         );

src/i18n/strings/en_EN.json

@@ -976,7 +976,11 @@
     "Match system theme": "Match system theme",
     "Use a system font": "Use a system font",
     "System font name": "System font name",
-    "Allow Peer-to-Peer for 1:1 calls (if you enable this, the other party might be able to see your IP address)": "Allow Peer-to-Peer for 1:1 calls (if you enable this, the other party might be able to see your IP address)",
+    "Allow Peer-to-Peer for 1:1 calls": "Allow Peer-to-Peer for 1:1 calls",
+    "When enabled, the other party might be able to see your IP address": "When enabled, the other party might be able to see your IP address",
+    "Automatic gain control": "Automatic gain control",
+    "Echo cancellation": "Echo cancellation",
+    "Noise suppression": "Noise suppression",
     "Send analytics data": "Send analytics data",
     "Record the client name, version, and url to recognise sessions more easily in session manager": "Record the client name, version, and url to recognise sessions more easily in session manager",
     "Never send encrypted messages to unverified sessions from this session": "Never send encrypted messages to unverified sessions from this session",
@@ -992,7 +996,8 @@
     "Show shortcut to welcome checklist above the room list": "Show shortcut to welcome checklist above the room list",
     "Show hidden events in timeline": "Show hidden events in timeline",
     "Low bandwidth mode (requires compatible homeserver)": "Low bandwidth mode (requires compatible homeserver)",
-    "Allow fallback call assist server turn.matrix.org when your homeserver does not offer one (your IP address would be shared during a call)": "Allow fallback call assist server turn.matrix.org when your homeserver does not offer one (your IP address would be shared during a call)",
+    "Allow fallback call assist server (turn.matrix.org)": "Allow fallback call assist server (turn.matrix.org)",
+    "Only applies if your homeserver does not offer one. Your IP address would be shared during a call.": "Only applies if your homeserver does not offer one. Your IP address would be shared during a call.",
     "Show previews/thumbnails for images": "Show previews/thumbnails for images",
     "Enable message search in encrypted rooms": "Enable message search in encrypted rooms",
     "How fast should messages be downloaded.": "How fast should messages be downloaded.",
@@ -1619,6 +1624,11 @@
     "No Microphones detected": "No Microphones detected",
     "Camera": "Camera",
     "No Webcams detected": "No Webcams detected",
+    "Voice settings": "Voice settings",
+    "Automatically adjust the microphone volume": "Automatically adjust the microphone volume",
+    "Video settings": "Video settings",
+    "Voice processing": "Voice processing",
+    "Connection": "Connection",
     "This room is not accessible by remote Matrix servers": "This room is not accessible by remote Matrix servers",
     "<b>Warning</b>: Upgrading a room will <i>not automatically migrate room members to the new version of the room.</i> We'll post a link to the new room in the old version of the room - room members will have to click this link to join the new room.": "<b>Warning</b>: Upgrading a room will <i>not automatically migrate room members to the new version of the room.</i> We'll post a link to the new room in the old version of the room - room members will have to click this link to join the new room.",
     "Upgrade this space to the recommended room version": "Upgrade this space to the recommended room version",

src/settings/Settings.tsx

@@ -127,7 +127,8 @@ export type SettingValueType = boolean |
     string |
     number[] |
     string[] |
-    Record<string, unknown>;
+    Record<string, unknown> |
+    null;
 
 export interface IBaseSetting<T extends SettingValueType = SettingValueType> {
     isFeature?: false | undefined;
@@ -712,10 +713,8 @@ export const SETTINGS: {[setting: string]: ISetting} = {
     },
     "webRtcAllowPeerToPeer": {
         supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS_WITH_CONFIG,
-        displayName: _td(
+        displayName: _td("Allow Peer-to-Peer for 1:1 calls"),
-            "Allow Peer-to-Peer for 1:1 calls " +
+        description: _td("When enabled, the other party might be able to see your IP address"),
-            "(if you enable this, the other party might be able to see your IP address)",
-        ),
         default: true,
         invertedSettingName: 'webRtcForceTURN',
     },
@@ -731,6 +730,21 @@ export const SETTINGS: {[setting: string]: ISetting} = {
         supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
         default: "default",
     },
+    "webrtc_audio_autoGainControl": {
+        supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
+        displayName: _td("Automatic gain control"),
+        default: true,
+    },
+    "webrtc_audio_echoCancellation": {
+        supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
+        displayName: _td("Echo cancellation"),
+        default: true,
+    },
+    "webrtc_audio_noiseSuppression": {
+        supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
+        displayName: _td("Noise suppression"),
+        default: true,
+    },
     "language": {
         supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS_WITH_CONFIG,
         default: "en",
@@ -902,9 +916,10 @@ export const SETTINGS: {[setting: string]: ISetting} = {
     },
     "fallbackICEServerAllowed": {
         supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS,
-        displayName: _td(
+        displayName: _td("Allow fallback call assist server (turn.matrix.org)"),
-            "Allow fallback call assist server turn.matrix.org when your homeserver " +
+        description: _td(
-            "does not offer one (your IP address would be shared during a call)",
+            "Only applies if your homeserver does not offer one. " +
+            "Your IP address would be shared during a call.",
         ),
         // This is a tri-state value, where `null` means "prompt the user".
         default: null,

test/MediaDeviceHandler-test.ts

@@ -0,0 +1,65 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { mocked } from 'jest-mock';
+
+import { SettingLevel } from "../src/settings/SettingLevel";
+import { MatrixClientPeg } from '../src/MatrixClientPeg';
+import { stubClient } from "./test-utils";
+import MediaDeviceHandler from "../src/MediaDeviceHandler";
+import SettingsStore from '../src/settings/SettingsStore';
+
+jest.mock("../src/settings/SettingsStore");
+
+const SettingsStoreMock = mocked(SettingsStore);
+
+describe("MediaDeviceHandler", () => {
+    beforeEach(() => {
+        stubClient();
+    });
+
+    afterEach(() => {
+        jest.clearAllMocks();
+    });
+
+    it("sets audio settings", async () => {
+        const expectedAudioSettings = new Map<string, boolean>([
+            ["webrtc_audio_autoGainControl", false],
+            ["webrtc_audio_echoCancellation", true],
+            ["webrtc_audio_noiseSuppression", false],
+        ]);
+
+        SettingsStoreMock.getValue.mockImplementation((settingName): any => {
+            return expectedAudioSettings.get(settingName);
+        });
+
+        await MediaDeviceHandler.setAudioAutoGainControl(false);
+        await MediaDeviceHandler.setAudioEchoCancellation(true);
+        await MediaDeviceHandler.setAudioNoiseSuppression(false);
+
+        expectedAudioSettings.forEach((value, key) => {
+            expect(SettingsStoreMock.setValue).toHaveBeenCalledWith(
+                key, null, SettingLevel.DEVICE, value,
+            );
+        });
+
+        expect(MatrixClientPeg.get().getMediaHandler().setAudioSettings).toHaveBeenCalledWith({
+            autoGainControl: false,
+            echoCancellation: true,
+            noiseSuppression: false,
+        });
+    });
+});

test/components/views/settings/tabs/user/VoiceUserSettingsTab-test.tsx

@@ -0,0 +1,56 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from 'react';
+import { mocked } from 'jest-mock';
+import { render } from '@testing-library/react';
+
+import VoiceUserSettingsTab from '../../../../../../src/components/views/settings/tabs/user/VoiceUserSettingsTab';
+import MediaDeviceHandler from "../../../../../../src/MediaDeviceHandler";
+
+jest.mock("../../../../../../src/MediaDeviceHandler");
+const MediaDeviceHandlerMock = mocked(MediaDeviceHandler);
+
+describe('<VoiceUserSettingsTab />', () => {
+    const getComponent = (): React.ReactElement => (<VoiceUserSettingsTab />);
+
+    beforeEach(() => {
+        jest.clearAllMocks();
+    });
+
+    it('renders audio processing settings', () => {
+        const { getByTestId } = render(getComponent());
+        expect(getByTestId('voice-auto-gain')).toBeTruthy();
+        expect(getByTestId('voice-noise-suppression')).toBeTruthy();
+        expect(getByTestId('voice-echo-cancellation')).toBeTruthy();
+    });
+
+    it('sets and displays audio processing settings', () => {
+        MediaDeviceHandlerMock.getAudioAutoGainControl.mockReturnValue(false);
+        MediaDeviceHandlerMock.getAudioEchoCancellation.mockReturnValue(true);
+        MediaDeviceHandlerMock.getAudioNoiseSuppression.mockReturnValue(false);
+
+        const { getByRole } = render(getComponent());
+
+        getByRole("switch", { name: "Automatically adjust the microphone volume" }).click();
+        getByRole("switch", { name: "Noise suppression" }).click();
+        getByRole("switch", { name: "Echo cancellation" }).click();
+
+        expect(MediaDeviceHandler.setAudioAutoGainControl).toHaveBeenCalledWith(true);
+        expect(MediaDeviceHandler.setAudioEchoCancellation).toHaveBeenCalledWith(false);
+        expect(MediaDeviceHandler.setAudioNoiseSuppression).toHaveBeenCalledWith(true);
+    });
+});

test/test-utils/test-utils.ts

@@ -185,6 +185,7 @@ export function createTestClient(): MatrixClient {
         getMediaHandler: jest.fn().mockReturnValue({
             setVideoInput: jest.fn(),
             setAudioInput: jest.fn(),
+            setAudioSettings: jest.fn(),
         } as unknown as MediaHandler),
         uploadContent: jest.fn(),
         getEventMapper: () => (opts) => new MatrixEvent(opts),