From 339d65dff87dc173cc5b0800140a353ba5181c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Thu, 24 Nov 2022 12:15:10 +0100 Subject: [PATCH 01/19] Handle the raw ICE connection state in the views MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than just providing a coarse "connected" or "not connected" value now the views receive the raw ICE connection state. Combined with other properties this will make possible to show a finer grained status (like done in the WebUI), although for now just "connected" or "not connected" is still shown as before. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 55 ++++--------------- .../talk/adapters/ParticipantDisplayItem.java | 14 +++-- 2 files changed, 20 insertions(+), 49 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 81c1030d2..abff92925 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -2072,7 +2072,10 @@ public class CallActivity extends CallBaseActivity { updateSelfVideoViewPosition(); } - private void updateSelfVideoViewConnected(boolean connected) { + private void updateSelfVideoViewIceConnectionState(PeerConnection.IceConnectionState iceConnectionState) { + boolean connected = iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || + iceConnectionState == PeerConnection.IceConnectionState.COMPLETED; + // FIXME In voice only calls there is no video view, so the progress bar would appear floating in the middle of // nowhere. However, a way to signal that the local participant is not connected to the HPB is still need in // that case. @@ -2133,28 +2136,6 @@ public class CallActivity extends CallBaseActivity { } } - private void handlePeerConnected(String sessionId, String videoStreamType) { - String participantDisplayItemId = sessionId + "-" + videoStreamType; - - if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { - updateSelfVideoViewConnected(true); - } else if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setConnected(true); - participantsAdapter.notifyDataSetChanged(); - } - } - - private void handlePeerDisconnected(String sessionId, String videoStreamType) { - String participantDisplayItemId = sessionId + "-" + videoStreamType; - - if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { - updateSelfVideoViewConnected(false); - } else if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setConnected(false); - participantsAdapter.notifyDataSetChanged(); - } - } - private void startSendingNick() { DataChannelMessage dataChannelMessage = new DataChannelMessage(); dataChannelMessage.setType("nickChanged"); @@ -2211,11 +2192,9 @@ public class CallActivity extends CallBaseActivity { PeerConnectionWrapper peerConnectionWrapper = getPeerConnectionWrapperForSessionIdAndType(session, videoStreamType); - boolean connected = false; + PeerConnection.IceConnectionState iceConnectionState = null; if (peerConnectionWrapper != null) { - PeerConnection.IceConnectionState iceConnectionState = peerConnectionWrapper.getPeerConnection().iceConnectionState(); - connected = iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || - iceConnectionState == PeerConnection.IceConnectionState.COMPLETED; + iceConnectionState = peerConnectionWrapper.getPeerConnection().iceConnectionState(); } String nick; @@ -2232,7 +2211,7 @@ public class CallActivity extends CallBaseActivity { ParticipantDisplayItem participantDisplayItem = new ParticipantDisplayItem(baseUrl, userId, session, - connected, + iceConnectionState, nick, defaultGuestNick, mediaStream, @@ -2692,19 +2671,11 @@ public class CallActivity extends CallBaseActivity { @Override public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) { runOnUiThread(() -> { - if (iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || - iceConnectionState == PeerConnection.IceConnectionState.COMPLETED) { - handlePeerConnected(sessionId, videoStreamType); - - return; - } - - if (iceConnectionState == PeerConnection.IceConnectionState.DISCONNECTED || - iceConnectionState == PeerConnection.IceConnectionState.NEW || - iceConnectionState == PeerConnection.IceConnectionState.CHECKING) { - handlePeerDisconnected(sessionId, videoStreamType); - - return; + if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { + updateSelfVideoViewIceConnectionState(iceConnectionState); + } else if (participantDisplayItems.get(participantDisplayItemId) != null) { + participantDisplayItems.get(participantDisplayItemId).setIceConnectionState(iceConnectionState); + participantsAdapter.notifyDataSetChanged(); } if (iceConnectionState == PeerConnection.IceConnectionState.CLOSED) { @@ -2718,8 +2689,6 @@ public class CallActivity extends CallBaseActivity { setCallState(CallStatus.PUBLISHER_FAILED); webSocketClient.clearResumeId(); hangup(false); - } else { - handlePeerDisconnected(sessionId, videoStreamType); } return; diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index 34fa963ac..63a0d3ef5 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -6,12 +6,13 @@ import com.nextcloud.talk.utils.ApiUtils; import org.webrtc.EglBase; import org.webrtc.MediaStream; +import org.webrtc.PeerConnection; public class ParticipantDisplayItem { private String baseUrl; private String userId; private String session; - private boolean connected; + private PeerConnection.IceConnectionState iceConnectionState; private String nick; private final String defaultGuestNick; private String urlForAvatar; @@ -21,11 +22,11 @@ public class ParticipantDisplayItem { private EglBase rootEglBase; private boolean isAudioEnabled; - public ParticipantDisplayItem(String baseUrl, String userId, String session, boolean connected, String nick, String defaultGuestNick, MediaStream mediaStream, String streamType, boolean streamEnabled, EglBase rootEglBase) { + public ParticipantDisplayItem(String baseUrl, String userId, String session, PeerConnection.IceConnectionState iceConnectionState, String nick, String defaultGuestNick, MediaStream mediaStream, String streamType, boolean streamEnabled, EglBase rootEglBase) { this.baseUrl = baseUrl; this.userId = userId; this.session = session; - this.connected = connected; + this.iceConnectionState = iceConnectionState; this.nick = nick; this.defaultGuestNick = defaultGuestNick; this.mediaStream = mediaStream; @@ -55,11 +56,12 @@ public class ParticipantDisplayItem { } public boolean isConnected() { - return connected; + return iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || + iceConnectionState == PeerConnection.IceConnectionState.COMPLETED; } - public void setConnected(boolean connected) { - this.connected = connected; + public void setIceConnectionState(PeerConnection.IceConnectionState iceConnectionState) { + this.iceConnectionState = iceConnectionState; } public String getNick() { From e887fde2a349e33bb5f7fc195ccbf001c6d07b46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 25 Nov 2022 13:12:42 +0100 Subject: [PATCH 02/19] Remove unused getters and setters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note that the session ID, user ID and the stream type attributes are still kept, as they can be useful to identify the instance when debugging. Signed-off-by: Daniel Calviño Sánchez --- .../talk/adapters/ParticipantDisplayItem.java | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index 63a0d3ef5..fa980ad5d 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -37,24 +37,12 @@ public class ParticipantDisplayItem { this.updateUrlForAvatar(); } - public String getUserId() { - return userId; - } - public void setUserId(String userId) { this.userId = userId; this.updateUrlForAvatar(); } - public String getSession() { - return session; - } - - public void setSession(String session) { - this.session = session; - } - public boolean isConnected() { return iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || iceConnectionState == PeerConnection.IceConnectionState.COMPLETED; @@ -98,14 +86,6 @@ public class ParticipantDisplayItem { this.mediaStream = mediaStream; } - public String getStreamType() { - return streamType; - } - - public void setStreamType(String streamType) { - this.streamType = streamType; - } - public boolean isStreamEnabled() { return streamEnabled; } @@ -118,10 +98,6 @@ public class ParticipantDisplayItem { return rootEglBase; } - public void setRootEglBase(EglBase rootEglBase) { - this.rootEglBase = rootEglBase; - } - public boolean isAudioEnabled() { return isAudioEnabled; } From 5fe9154c9ae3cedbf985ba242b780484fd8e2f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 25 Nov 2022 13:13:27 +0100 Subject: [PATCH 03/19] Declare attributes set just once in constructor as "final" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../nextcloud/talk/adapters/ParticipantDisplayItem.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index fa980ad5d..b9744a8f8 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -9,17 +9,17 @@ import org.webrtc.MediaStream; import org.webrtc.PeerConnection; public class ParticipantDisplayItem { - private String baseUrl; + private final String baseUrl; private String userId; - private String session; + private final String session; private PeerConnection.IceConnectionState iceConnectionState; private String nick; private final String defaultGuestNick; private String urlForAvatar; private MediaStream mediaStream; - private String streamType; + private final String streamType; private boolean streamEnabled; - private EglBase rootEglBase; + private final EglBase rootEglBase; private boolean isAudioEnabled; public ParticipantDisplayItem(String baseUrl, String userId, String session, PeerConnection.IceConnectionState iceConnectionState, String nick, String defaultGuestNick, MediaStream mediaStream, String streamType, boolean streamEnabled, EglBase rootEglBase) { From d67b04dff8aec30ab56dfaf93db28670b304802e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 25 Nov 2022 13:14:46 +0100 Subject: [PATCH 04/19] Reorder attributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generic final attributes first, followed by object specific final attributes and then other object attributes. Signed-off-by: Daniel Calviño Sánchez --- .../talk/adapters/ParticipantDisplayItem.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index b9744a8f8..1ababf417 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -10,16 +10,18 @@ import org.webrtc.PeerConnection; public class ParticipantDisplayItem { private final String baseUrl; - private String userId; + private final String defaultGuestNick; + private final EglBase rootEglBase; + private final String session; + private final String streamType; + + private String userId; private PeerConnection.IceConnectionState iceConnectionState; private String nick; - private final String defaultGuestNick; private String urlForAvatar; private MediaStream mediaStream; - private final String streamType; private boolean streamEnabled; - private final EglBase rootEglBase; private boolean isAudioEnabled; public ParticipantDisplayItem(String baseUrl, String userId, String session, PeerConnection.IceConnectionState iceConnectionState, String nick, String defaultGuestNick, MediaStream mediaStream, String streamType, boolean streamEnabled, EglBase rootEglBase) { From 8a316d94f545f74deb78487d43f758fe81aa4fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 25 Nov 2022 20:05:21 +0100 Subject: [PATCH 05/19] Notify that data set changed automatically when display item changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of explicitly calling "notifyDataSetChanged" after setting values on a ParticipantDisplayItem now the adapter observes all its items and calls "notifyDataSetChanged" automatically when any of them changes. Although this adds some boilerplate code it will make possible to update the ParticipantDisplayItems and automatically propagate the changes to the adapter when a model changes, rather than having to explicitly do it from the CallActivity. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 23 ++------ .../talk/adapters/ParticipantDisplayItem.java | 27 +++++++++ .../ParticipantDisplayItemNotifier.java | 55 +++++++++++++++++++ .../talk/adapters/ParticipantsAdapter.java | 11 ++++ 4 files changed, 97 insertions(+), 19 deletions(-) create mode 100644 app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItemNotifier.java diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index abff92925..1fd0d2c99 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -740,6 +740,10 @@ public class CallActivity extends CallBaseActivity { } }); + if (participantsAdapter != null) { + participantsAdapter.destroy(); + } + participantsAdapter = new ParticipantsAdapter( this, participantDisplayItems, @@ -1853,17 +1857,11 @@ public class CallActivity extends CallBaseActivity { String userId = userIdsBySessionId.get(sessionId); if (userId != null) { runOnUiThread(() -> { - boolean notifyDataSetChanged = false; if (participantDisplayItems.get(sessionId + "-video") != null) { participantDisplayItems.get(sessionId + "-video").setUserId(userId); - notifyDataSetChanged = true; } if (participantDisplayItems.get(sessionId + "-screen") != null) { participantDisplayItems.get(sessionId + "-screen").setUserId(userId); - notifyDataSetChanged = true; - } - if (notifyDataSetChanged) { - participantsAdapter.notifyDataSetChanged(); } }); } @@ -2527,17 +2525,11 @@ public class CallActivity extends CallBaseActivity { private void onOfferOrAnswer(String nick) { this.nick = nick; - boolean notifyDataSetChanged = false; if (participantDisplayItems.get(sessionId + "-video") != null) { participantDisplayItems.get(sessionId + "-video").setNick(nick); - notifyDataSetChanged = true; } if (participantDisplayItems.get(sessionId + "-screen") != null) { participantDisplayItems.get(sessionId + "-screen").setNick(nick); - notifyDataSetChanged = true; - } - if (notifyDataSetChanged) { - participantsAdapter.notifyDataSetChanged(); } } @@ -2582,7 +2574,6 @@ public class CallActivity extends CallBaseActivity { runOnUiThread(() -> { if (participantDisplayItems.get(participantDisplayItemId) != null) { participantDisplayItems.get(participantDisplayItemId).setAudioEnabled(true); - participantsAdapter.notifyDataSetChanged(); } }); } @@ -2592,7 +2583,6 @@ public class CallActivity extends CallBaseActivity { runOnUiThread(() -> { if (participantDisplayItems.get(participantDisplayItemId) != null) { participantDisplayItems.get(participantDisplayItemId).setAudioEnabled(false); - participantsAdapter.notifyDataSetChanged(); } }); } @@ -2602,7 +2592,6 @@ public class CallActivity extends CallBaseActivity { runOnUiThread(() -> { if (participantDisplayItems.get(participantDisplayItemId) != null) { participantDisplayItems.get(participantDisplayItemId).setStreamEnabled(true); - participantsAdapter.notifyDataSetChanged(); } }); } @@ -2612,7 +2601,6 @@ public class CallActivity extends CallBaseActivity { runOnUiThread(() -> { if (participantDisplayItems.get(participantDisplayItemId) != null) { participantDisplayItems.get(participantDisplayItemId).setStreamEnabled(false); - participantsAdapter.notifyDataSetChanged(); } }); } @@ -2622,7 +2610,6 @@ public class CallActivity extends CallBaseActivity { runOnUiThread(() -> { if (participantDisplayItems.get(participantDisplayItemId) != null) { participantDisplayItems.get(participantDisplayItemId).setNick(nick); - participantsAdapter.notifyDataSetChanged(); } }); } @@ -2664,7 +2651,6 @@ public class CallActivity extends CallBaseActivity { ParticipantDisplayItem participantDisplayItem = participantDisplayItems.get(participantDisplayItemId); participantDisplayItem.setMediaStream(mediaStream); participantDisplayItem.setStreamEnabled(hasAtLeastOneVideoStream); - participantsAdapter.notifyDataSetChanged(); }); } @@ -2675,7 +2661,6 @@ public class CallActivity extends CallBaseActivity { updateSelfVideoViewIceConnectionState(iceConnectionState); } else if (participantDisplayItems.get(participantDisplayItemId) != null) { participantDisplayItems.get(participantDisplayItemId).setIceConnectionState(iceConnectionState); - participantsAdapter.notifyDataSetChanged(); } if (iceConnectionState == PeerConnection.IceConnectionState.CLOSED) { diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index 1ababf417..746f21710 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -9,6 +9,13 @@ import org.webrtc.MediaStream; import org.webrtc.PeerConnection; public class ParticipantDisplayItem { + + public interface Observer { + void onChange(); + } + + private final ParticipantDisplayItemNotifier participantDisplayItemNotifier = new ParticipantDisplayItemNotifier(); + private final String baseUrl; private final String defaultGuestNick; private final EglBase rootEglBase; @@ -43,6 +50,8 @@ public class ParticipantDisplayItem { this.userId = userId; this.updateUrlForAvatar(); + + participantDisplayItemNotifier.notifyChange(); } public boolean isConnected() { @@ -52,6 +61,8 @@ public class ParticipantDisplayItem { public void setIceConnectionState(PeerConnection.IceConnectionState iceConnectionState) { this.iceConnectionState = iceConnectionState; + + participantDisplayItemNotifier.notifyChange(); } public String getNick() { @@ -66,6 +77,8 @@ public class ParticipantDisplayItem { this.nick = nick; this.updateUrlForAvatar(); + + participantDisplayItemNotifier.notifyChange(); } public String getUrlForAvatar() { @@ -86,6 +99,8 @@ public class ParticipantDisplayItem { public void setMediaStream(MediaStream mediaStream) { this.mediaStream = mediaStream; + + participantDisplayItemNotifier.notifyChange(); } public boolean isStreamEnabled() { @@ -94,6 +109,8 @@ public class ParticipantDisplayItem { public void setStreamEnabled(boolean streamEnabled) { this.streamEnabled = streamEnabled; + + participantDisplayItemNotifier.notifyChange(); } public EglBase getRootEglBase() { @@ -106,6 +123,16 @@ public class ParticipantDisplayItem { public void setAudioEnabled(boolean audioEnabled) { isAudioEnabled = audioEnabled; + + participantDisplayItemNotifier.notifyChange(); + } + + public void addObserver(Observer observer) { + participantDisplayItemNotifier.addObserver(observer); + } + + public void removeObserver(Observer observer) { + participantDisplayItemNotifier.removeObserver(observer); } @Override diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItemNotifier.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItemNotifier.java new file mode 100644 index 000000000..239b9f85b --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItemNotifier.java @@ -0,0 +1,55 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.adapters; + +import com.nextcloud.talk.signaling.SignalingMessageReceiver; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Helper class to register and notify ParticipantDisplayItem.Observers. + * + * This class is only meant for internal use by ParticipantDisplayItem; observers must register themselves against a + * ParticipantDisplayItem rather than against a ParticipantDisplayItemNotifier. + */ +class ParticipantDisplayItemNotifier { + + private final Set participantDisplayItemObservers = new LinkedHashSet<>(); + + public synchronized void addObserver(ParticipantDisplayItem.Observer observer) { + if (observer == null) { + throw new IllegalArgumentException("ParticipantDisplayItem.Observer can not be null"); + } + + participantDisplayItemObservers.add(observer); + } + + public synchronized void removeObserver(ParticipantDisplayItem.Observer observer) { + participantDisplayItemObservers.remove(observer); + } + + public synchronized void notifyChange() { + for (ParticipantDisplayItem.Observer observer : new ArrayList<>(participantDisplayItemObservers)) { + observer.onChange(); + } + } +} diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java index b8b5cc60b..85e24d50d 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java @@ -29,6 +29,8 @@ public class ParticipantsAdapter extends BaseAdapter { private static final String TAG = "ParticipantsAdapter"; + private final ParticipantDisplayItem.Observer participantDisplayItemObserver = this::notifyDataSetChanged; + private final Context mContext; private final ArrayList participantDisplayItems; private final RelativeLayout gridViewWrapper; @@ -50,8 +52,17 @@ public class ParticipantsAdapter extends BaseAdapter { this.participantDisplayItems = new ArrayList<>(); this.participantDisplayItems.addAll(participantDisplayItems.values()); + + for (ParticipantDisplayItem participantDisplayItem : this.participantDisplayItems) { + participantDisplayItem.addObserver(participantDisplayItemObserver); + } } + public void destroy() { + for (ParticipantDisplayItem participantDisplayItem : participantDisplayItems) { + participantDisplayItem.removeObserver(participantDisplayItemObserver); + } + } @Override public int getCount() { From d72648379eefc2c9b2990e8cbfd65ce25400ba78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 25 Nov 2022 21:49:47 +0100 Subject: [PATCH 06/19] Add model for (remote) call participants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clients that modify the model would define the variables using the mutable subclass, while clients that only need to access the model are expected to use the read-only base class. The read-only class provides an observer; as it is expected that the model will be modified from background threads but observed from the main thread the observer can be registered along a handler to be notified on its thread, independently of on which thread the values were set. Currently there does not seem to be a need to observe each value on its own, so the observer is notified in a coarse way when any value changes. Signed-off-by: Daniel Calviño Sánchez --- .../talk/call/CallParticipantModel.java | 164 ++++++++++++++++++ .../call/CallParticipantModelNotifier.java | 86 +++++++++ .../call/MutableCallParticipantModel.java | 67 +++++++ 3 files changed, 317 insertions(+) create mode 100644 app/src/main/java/com/nextcloud/talk/call/CallParticipantModel.java create mode 100644 app/src/main/java/com/nextcloud/talk/call/CallParticipantModelNotifier.java create mode 100644 app/src/main/java/com/nextcloud/talk/call/MutableCallParticipantModel.java diff --git a/app/src/main/java/com/nextcloud/talk/call/CallParticipantModel.java b/app/src/main/java/com/nextcloud/talk/call/CallParticipantModel.java new file mode 100644 index 000000000..8c3947824 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/call/CallParticipantModel.java @@ -0,0 +1,164 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import android.os.Handler; + +import org.webrtc.MediaStream; +import org.webrtc.PeerConnection; + +import java.util.Objects; + +/** + * Read-only data model for (remote) call participants. + * + * The received audio and video are available only if the participant is sending them and also has them enabled. + * Before a connection is established it is not known whether audio and video are available or not, so null is returned + * in that case (therefore it should not be autoboxed to a plain boolean without checking that). + * + * Audio and video in screen shares, on the other hand, are always seen as available. + * + * Clients of the model can observe it with CallParticipantModel.Observer to be notified when any value changes. + * Getters called after receiving a notification are guaranteed to provide at least the value that triggered the + * notification, but it may return even a more up to date one (so getting the value again on the following + * notification may return the same value as before). + */ +public class CallParticipantModel { + + public interface Observer { + void onChange(); + } + + protected class Data { + + private T value; + + public T getValue() { + return value; + } + + public void setValue(T value) { + if (Objects.equals(this.value, value)) { + return; + } + + this.value = value; + + callParticipantModelNotifier.notifyChange(); + } + } + + private final CallParticipantModelNotifier callParticipantModelNotifier = new CallParticipantModelNotifier(); + + protected final String sessionId; + + protected Data userId; + protected Data nick; + + protected Data iceConnectionState; + protected Data mediaStream; + protected Data audioAvailable; + protected Data videoAvailable; + + protected Data screenIceConnectionState; + protected Data screenMediaStream; + + public CallParticipantModel(String sessionId) { + this.sessionId = sessionId; + + this.userId = new Data<>(); + this.nick = new Data<>(); + + this.iceConnectionState = new Data<>(); + this.mediaStream = new Data<>(); + this.audioAvailable = new Data<>(); + this.videoAvailable = new Data<>(); + + this.screenIceConnectionState = new Data<>(); + this.screenMediaStream = new Data<>(); + } + + public String getSessionId() { + return sessionId; + } + + public String getUserId() { + return userId.getValue(); + } + + public String getNick() { + return nick.getValue(); + } + + public PeerConnection.IceConnectionState getIceConnectionState() { + return iceConnectionState.getValue(); + } + + public MediaStream getMediaStream() { + return mediaStream.getValue(); + } + + public Boolean isAudioAvailable() { + return audioAvailable.getValue(); + } + + public Boolean isVideoAvailable() { + return videoAvailable.getValue(); + } + + public PeerConnection.IceConnectionState getScreenIceConnectionState() { + return screenIceConnectionState.getValue(); + } + + public MediaStream getScreenMediaStream() { + return screenMediaStream.getValue(); + } + + /** + * Adds an Observer to be notified when any value changes. + * + * @param observer the Observer + * @see CallParticipantModel#addObserver(Observer, Handler) + */ + public void addObserver(Observer observer) { + addObserver(observer, null); + } + + /** + * Adds an observer to be notified when any value changes. + * + * The observer will be notified on the thread associated to the given handler. If no handler is given the + * observer will be immediately notified on the same thread that changed the value; the observer will be + * immediately notified too if the thread of the handler is the same thread that changed the value. + * + * An observer is expected to be added only once. If the same observer is added again it will be notified just + * once on the thread of the last handler. + * + * @param observer the Observer + * @param handler a Handler for the thread to be notified on + */ + public void addObserver(Observer observer, Handler handler) { + callParticipantModelNotifier.addObserver(observer, handler); + } + + public void removeObserver(Observer observer) { + callParticipantModelNotifier.removeObserver(observer); + } +} diff --git a/app/src/main/java/com/nextcloud/talk/call/CallParticipantModelNotifier.java b/app/src/main/java/com/nextcloud/talk/call/CallParticipantModelNotifier.java new file mode 100644 index 000000000..ddf30c1d7 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/call/CallParticipantModelNotifier.java @@ -0,0 +1,86 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import android.os.Handler; +import android.os.Looper; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * Helper class to register and notify CallParticipantModel.Observers. + * + * This class is only meant for internal use by CallParticipantModel; observers must register themselves against a + * CallParticipantModel rather than against a CallParticipantModelNotifier. + */ +class CallParticipantModelNotifier { + + /** + * Helper class to associate a CallParticipantModel.Observer with a Handler. + */ + private static class CallParticipantModelObserverOn { + public final CallParticipantModel.Observer observer; + public final Handler handler; + + private CallParticipantModelObserverOn(CallParticipantModel.Observer observer, Handler handler) { + this.observer = observer; + this.handler = handler; + } + } + + private final List callParticipantModelObserversOn = new ArrayList<>(); + + public synchronized void addObserver(CallParticipantModel.Observer observer, Handler handler) { + if (observer == null) { + throw new IllegalArgumentException("CallParticipantModel.Observer can not be null"); + } + + removeObserver(observer); + + callParticipantModelObserversOn.add(new CallParticipantModelObserverOn(observer, handler)); + } + + public synchronized void removeObserver(CallParticipantModel.Observer observer) { + Iterator it = callParticipantModelObserversOn.iterator(); + while (it.hasNext()) { + CallParticipantModelObserverOn observerOn = it.next(); + + if (observerOn.observer == observer) { + it.remove(); + + return; + } + } + } + + public synchronized void notifyChange() { + for (CallParticipantModelObserverOn observerOn : new ArrayList<>(callParticipantModelObserversOn)) { + if (observerOn.handler == null || observerOn.handler.getLooper() == Looper.myLooper()) { + observerOn.observer.onChange(); + } else { + observerOn.handler.post(() -> { + observerOn.observer.onChange(); + }); + } + } + } +} diff --git a/app/src/main/java/com/nextcloud/talk/call/MutableCallParticipantModel.java b/app/src/main/java/com/nextcloud/talk/call/MutableCallParticipantModel.java new file mode 100644 index 000000000..4023bd296 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/call/MutableCallParticipantModel.java @@ -0,0 +1,67 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import org.webrtc.MediaStream; +import org.webrtc.PeerConnection; + +/** + * Mutable data model for (remote) call participants. + * + * There is no synchronization when setting the values; if needed, it should be handled by the clients of the model. + */ +public class MutableCallParticipantModel extends CallParticipantModel { + + public MutableCallParticipantModel(String sessionId) { + super(sessionId); + } + + public void setUserId(String userId) { + this.userId.setValue(userId); + } + + public void setNick(String nick) { + this.nick.setValue(nick); + } + + public void setIceConnectionState(PeerConnection.IceConnectionState iceConnectionState) { + this.iceConnectionState.setValue(iceConnectionState); + } + + public void setMediaStream(MediaStream mediaStream) { + this.mediaStream.setValue(mediaStream); + } + + public void setAudioAvailable(Boolean audioAvailable) { + this.audioAvailable.setValue(audioAvailable); + } + + public void setVideoAvailable(Boolean videoAvailable) { + this.videoAvailable.setValue(videoAvailable); + } + + public void setScreenIceConnectionState(PeerConnection.IceConnectionState screenIceConnectionState) { + this.screenIceConnectionState.setValue(screenIceConnectionState); + } + + public void setScreenMediaStream(MediaStream screenMediaStream) { + this.screenMediaStream.setValue(screenMediaStream); + } +} From 18f21c4f482a573b32173966d749c89e2c144316 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Fri, 25 Nov 2022 22:07:05 +0100 Subject: [PATCH 07/19] Update ParticipantDisplayItem from CallParticipantModel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of explicitly setting the values on the ParticipantDisplayItems now the values are set on the CallParticipantModels, and the items are automatically updated from their model when they change. Different items are still used for the audio/video and screen shares of the same participant, so the type is used to select from which properties of the model is the item updated. As the model may be updated from background threads it is explicitly observed by the items from the main thread using a Handler shared by all the items. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 189 +++++++++--------- .../talk/adapters/ParticipantDisplayItem.java | 115 ++++++----- 2 files changed, 150 insertions(+), 154 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 1fd0d2c99..321b8257b 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -60,6 +60,8 @@ import com.nextcloud.talk.adapters.ParticipantDisplayItem; import com.nextcloud.talk.adapters.ParticipantsAdapter; import com.nextcloud.talk.api.NcApi; import com.nextcloud.talk.application.NextcloudTalkApplication; +import com.nextcloud.talk.call.CallParticipantModel; +import com.nextcloud.talk.call.MutableCallParticipantModel; import com.nextcloud.talk.data.user.model.User; import com.nextcloud.talk.databinding.CallActivityBinding; import com.nextcloud.talk.events.ConfigurationChangeEvent; @@ -231,7 +233,6 @@ public class CallActivity extends CallBaseActivity { private MediaStream localStream; private String credentials; private List peerConnectionWrapperList = new ArrayList<>(); - private Map userIdsBySessionId = new HashMap<>(); private boolean videoOn = false; private boolean microphoneOn = false; @@ -267,6 +268,8 @@ public class CallActivity extends CallBaseActivity { private Map peerConnectionObservers = new HashMap<>(); + private Map callParticipantModels = new HashMap<>(); + private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener = new SignalingMessageReceiver.ParticipantListMessageListener() { @Override @@ -382,6 +385,7 @@ public class CallActivity extends CallBaseActivity { requestBluetoothPermission(); } basicInitialization(); + callParticipantModels = new HashMap<>(); participantDisplayItems = new HashMap<>(); initViews(); if (!isConnectionEstablished()) { @@ -1776,7 +1780,7 @@ public class CallActivity extends CallBaseActivity { Log.d(TAG, "processUsersInRoom"); List newSessions = new ArrayList<>(); Set oldSessions = new HashSet<>(); - userIdsBySessionId = new HashMap<>(); + Map userIdsBySessionId = new HashMap<>(); hasMCU = hasExternalSignalingServer && webSocketClient != null && webSocketClient.hasMCU(); Log.d(TAG, " hasMCU is " + hasMCU); @@ -1856,15 +1860,16 @@ public class CallActivity extends CallBaseActivity { String userId = userIdsBySessionId.get(sessionId); if (userId != null) { - runOnUiThread(() -> { - if (participantDisplayItems.get(sessionId + "-video") != null) { - participantDisplayItems.get(sessionId + "-video").setUserId(userId); - } - if (participantDisplayItems.get(sessionId + "-screen") != null) { - participantDisplayItems.get(sessionId + "-screen").setUserId(userId); - } - }); + callParticipantModels.get(sessionId).setUserId(userId); } + + String nick; + if (hasExternalSignalingServer) { + nick = webSocketClient.getDisplayNameForSession(sessionId); + } else { + nick = offerAnswerNickProviders.get(sessionId) != null ? offerAnswerNickProviders.get(sessionId).getNick() : ""; + } + callParticipantModels.get(sessionId).setNick(nick); } if (newSessions.size() > 0 && currentCallStatus != CallStatus.IN_CONVERSATION) { @@ -1991,14 +1996,16 @@ public class CallActivity extends CallBaseActivity { peerConnectionWrapper.addObserver(peerConnectionObserver); if (!publisher) { + MutableCallParticipantModel mutableCallParticipantModel = callParticipantModels.get(sessionId); + if (mutableCallParticipantModel == null) { + mutableCallParticipantModel = new MutableCallParticipantModel(sessionId); + callParticipantModels.put(sessionId, mutableCallParticipantModel); + } + + final CallParticipantModel callParticipantModel = mutableCallParticipantModel; + runOnUiThread(() -> { - // userId is unknown here, but it will be got based on the session id, and the stream will be - // updated once it is added to the connection. - setupVideoStreamForLayout( - null, - sessionId, - false, - type); + setupVideoStreamForLayout(callParticipantModel, type); }); } @@ -2036,6 +2043,20 @@ public class CallActivity extends CallBaseActivity { peerConnectionWrapper.removeObserver(peerConnectionObserver); runOnUiThread(() -> removeMediaStream(sessionId, videoStreamType)); + + MutableCallParticipantModel mutableCallParticipantModel = callParticipantModels.get(sessionId); + if (mutableCallParticipantModel != null) { + if ("screen".equals(videoStreamType)) { + mutableCallParticipantModel.setScreenMediaStream(null); + mutableCallParticipantModel.setScreenIceConnectionState(null); + } else { + mutableCallParticipantModel.setMediaStream(null); + mutableCallParticipantModel.setIceConnectionState(null); + mutableCallParticipantModel.setAudioAvailable(null); + mutableCallParticipantModel.setVideoAvailable(null); + } + } + deletePeerConnection(peerConnectionWrapper); } } @@ -2051,12 +2072,19 @@ public class CallActivity extends CallBaseActivity { signalingMessageReceiver.removeListener(offerAnswerNickProvider.getVideoWebRtcMessageListener()); signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener()); } + + callParticipantModels.remove(sessionId); } } private void removeMediaStream(String sessionId, String videoStreamType) { Log.d(TAG, "removeMediaStream"); - participantDisplayItems.remove(sessionId + "-" + videoStreamType); + ParticipantDisplayItem participantDisplayItem = participantDisplayItems.remove(sessionId + "-" + videoStreamType); + if (participantDisplayItem == null) { + return; + } + + participantDisplayItem.destroy(); if (!isDestroyed()) { initGridAdapter(); @@ -2183,40 +2211,16 @@ public class CallActivity extends CallBaseActivity { this); } - private void setupVideoStreamForLayout(@Nullable MediaStream mediaStream, - String session, - boolean videoStreamEnabled, - String videoStreamType) { - PeerConnectionWrapper peerConnectionWrapper = getPeerConnectionWrapperForSessionIdAndType(session, - videoStreamType); - - PeerConnection.IceConnectionState iceConnectionState = null; - if (peerConnectionWrapper != null) { - iceConnectionState = peerConnectionWrapper.getPeerConnection().iceConnectionState(); - } - - String nick; - if (hasExternalSignalingServer) { - nick = webSocketClient.getDisplayNameForSession(session); - } else { - nick = offerAnswerNickProviders.get(session) != null ? offerAnswerNickProviders.get(session).getNick() : ""; - } - - String userId = userIdsBySessionId.get(session); - + private void setupVideoStreamForLayout(CallParticipantModel callParticipantModel, String videoStreamType) { String defaultGuestNick = getResources().getString(R.string.nc_nick_guest); ParticipantDisplayItem participantDisplayItem = new ParticipantDisplayItem(baseUrl, - userId, - session, - iceConnectionState, - nick, defaultGuestNick, - mediaStream, + rootEglBase, videoStreamType, - videoStreamEnabled, - rootEglBase); - participantDisplayItems.put(session + "-" + videoStreamType, participantDisplayItem); + callParticipantModel); + String sessionId = callParticipantModel.getSessionId(); + participantDisplayItems.put(sessionId + "-" + videoStreamType, participantDisplayItem); initGridAdapter(); } @@ -2525,11 +2529,8 @@ public class CallActivity extends CallBaseActivity { private void onOfferOrAnswer(String nick) { this.nick = nick; - if (participantDisplayItems.get(sessionId + "-video") != null) { - participantDisplayItems.get(sessionId + "-video").setNick(nick); - } - if (participantDisplayItems.get(sessionId + "-screen") != null) { - participantDisplayItems.get(sessionId + "-screen").setNick(nick); + if (callParticipantModels.get(sessionId) != null) { + callParticipantModels.get(sessionId).setNick(nick); } } @@ -2562,56 +2563,45 @@ public class CallActivity extends CallBaseActivity { private class CallActivityDataChannelMessageListener implements PeerConnectionWrapper.DataChannelMessageListener { - private final String participantDisplayItemId; + private final String sessionId; private CallActivityDataChannelMessageListener(String sessionId) { - // DataChannel messages are sent only in video peers, so the listener only acts on the "video" items. - this.participantDisplayItemId = sessionId + "-video"; + this.sessionId = sessionId; } @Override public void onAudioOn() { - runOnUiThread(() -> { - if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setAudioEnabled(true); - } - }); + if (callParticipantModels.get(sessionId) != null) { + callParticipantModels.get(sessionId).setAudioAvailable(true); + } } @Override public void onAudioOff() { - runOnUiThread(() -> { - if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setAudioEnabled(false); - } - }); + if (callParticipantModels.get(sessionId) != null) { + callParticipantModels.get(sessionId).setAudioAvailable(false); + } } @Override public void onVideoOn() { - runOnUiThread(() -> { - if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setStreamEnabled(true); - } - }); + if (callParticipantModels.get(sessionId) != null) { + callParticipantModels.get(sessionId).setVideoAvailable(true); + } } @Override public void onVideoOff() { - runOnUiThread(() -> { - if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setStreamEnabled(false); - } - }); + if (callParticipantModels.get(sessionId) != null) { + callParticipantModels.get(sessionId).setVideoAvailable(false); + } } @Override public void onNickChanged(String nick) { - runOnUiThread(() -> { - if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setNick(nick); - } - }); + if (callParticipantModels.get(sessionId) != null) { + callParticipantModels.get(sessionId).setNick(nick); + } } } @@ -2619,12 +2609,10 @@ public class CallActivity extends CallBaseActivity { private final String sessionId; private final String videoStreamType; - private final String participantDisplayItemId; private CallActivityPeerConnectionObserver(String sessionId, String videoStreamType) { this.sessionId = sessionId; this.videoStreamType = videoStreamType; - this.participantDisplayItemId = sessionId + "-" + videoStreamType; } @Override @@ -2638,29 +2626,38 @@ public class CallActivity extends CallBaseActivity { } private void handleStream(MediaStream mediaStream) { - runOnUiThread(() -> { - if (participantDisplayItems.get(participantDisplayItemId) == null) { - return; - } + if (callParticipantModels.get(sessionId) == null) { + return; + } - boolean hasAtLeastOneVideoStream = false; - if (mediaStream != null) { - hasAtLeastOneVideoStream = mediaStream.videoTracks != null && mediaStream.videoTracks.size() > 0; - } + if ("screen".equals(videoStreamType)) { + callParticipantModels.get(sessionId).setScreenMediaStream(mediaStream); - ParticipantDisplayItem participantDisplayItem = participantDisplayItems.get(participantDisplayItemId); - participantDisplayItem.setMediaStream(mediaStream); - participantDisplayItem.setStreamEnabled(hasAtLeastOneVideoStream); - }); + return; + } + + boolean hasAtLeastOneVideoStream = false; + if (mediaStream != null) { + hasAtLeastOneVideoStream = mediaStream.videoTracks != null && mediaStream.videoTracks.size() > 0; + } + + callParticipantModels.get(sessionId).setMediaStream(mediaStream); + callParticipantModels.get(sessionId).setVideoAvailable(hasAtLeastOneVideoStream); } @Override public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) { + if (callParticipantModels.get(sessionId) != null) { + if ("screen".equals(videoStreamType)) { + callParticipantModels.get(sessionId).setScreenIceConnectionState(iceConnectionState); + } else { + callParticipantModels.get(sessionId).setIceConnectionState(iceConnectionState); + } + } + runOnUiThread(() -> { if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { updateSelfVideoViewIceConnectionState(iceConnectionState); - } else if (participantDisplayItems.get(participantDisplayItemId) != null) { - participantDisplayItems.get(participantDisplayItemId).setIceConnectionState(iceConnectionState); } if (iceConnectionState == PeerConnection.IceConnectionState.CLOSED) { diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index 746f21710..6a9912ead 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -1,7 +1,10 @@ package com.nextcloud.talk.adapters; +import android.os.Handler; +import android.os.Looper; import android.text.TextUtils; +import com.nextcloud.talk.call.CallParticipantModel; import com.nextcloud.talk.utils.ApiUtils; import org.webrtc.EglBase; @@ -14,6 +17,11 @@ public class ParticipantDisplayItem { void onChange(); } + /** + * Shared handler to receive change notifications from the model on the main thread. + */ + private static final Handler handler = new Handler(Looper.getMainLooper()); + private final ParticipantDisplayItemNotifier participantDisplayItemNotifier = new ParticipantDisplayItemNotifier(); private final String baseUrl; @@ -23,6 +31,10 @@ public class ParticipantDisplayItem { private final String session; private final String streamType; + private final CallParticipantModel callParticipantModel; + + private final CallParticipantModel.Observer callParticipantModelObserver = this::updateFromModel; + private String userId; private PeerConnection.IceConnectionState iceConnectionState; private String nick; @@ -31,60 +43,48 @@ public class ParticipantDisplayItem { private boolean streamEnabled; private boolean isAudioEnabled; - public ParticipantDisplayItem(String baseUrl, String userId, String session, PeerConnection.IceConnectionState iceConnectionState, String nick, String defaultGuestNick, MediaStream mediaStream, String streamType, boolean streamEnabled, EglBase rootEglBase) { + public ParticipantDisplayItem(String baseUrl, String defaultGuestNick, EglBase rootEglBase, String streamType, + CallParticipantModel callParticipantModel) { this.baseUrl = baseUrl; - this.userId = userId; - this.session = session; - this.iceConnectionState = iceConnectionState; - this.nick = nick; this.defaultGuestNick = defaultGuestNick; - this.mediaStream = mediaStream; - this.streamType = streamType; - this.streamEnabled = streamEnabled; this.rootEglBase = rootEglBase; - this.updateUrlForAvatar(); + this.session = callParticipantModel.getSessionId(); + this.streamType = streamType; + + this.callParticipantModel = callParticipantModel; + this.callParticipantModel.addObserver(callParticipantModelObserver, handler); + + updateFromModel(); } - public void setUserId(String userId) { - this.userId = userId; + public void destroy() { + this.callParticipantModel.removeObserver(callParticipantModelObserver); + } + + private void updateFromModel() { + userId = callParticipantModel.getUserId(); + nick = callParticipantModel.getNick(); this.updateUrlForAvatar(); - participantDisplayItemNotifier.notifyChange(); - } - - public boolean isConnected() { - return iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || - iceConnectionState == PeerConnection.IceConnectionState.COMPLETED; - } - - public void setIceConnectionState(PeerConnection.IceConnectionState iceConnectionState) { - this.iceConnectionState = iceConnectionState; - - participantDisplayItemNotifier.notifyChange(); - } - - public String getNick() { - if (TextUtils.isEmpty(userId) && TextUtils.isEmpty(nick)) { - return defaultGuestNick; + if ("screen".equals(streamType)) { + iceConnectionState = callParticipantModel.getScreenIceConnectionState(); + mediaStream = callParticipantModel.getScreenMediaStream(); + isAudioEnabled = true; + streamEnabled = true; + } else { + iceConnectionState = callParticipantModel.getIceConnectionState(); + mediaStream = callParticipantModel.getMediaStream(); + isAudioEnabled = callParticipantModel.isAudioAvailable() != null ? + callParticipantModel.isAudioAvailable() : false; + streamEnabled = callParticipantModel.isVideoAvailable() != null ? + callParticipantModel.isVideoAvailable() : false; } - return nick; - } - - public void setNick(String nick) { - this.nick = nick; - - this.updateUrlForAvatar(); - participantDisplayItemNotifier.notifyChange(); } - public String getUrlForAvatar() { - return urlForAvatar; - } - private void updateUrlForAvatar() { if (!TextUtils.isEmpty(userId)) { urlForAvatar = ApiUtils.getUrlForAvatar(baseUrl, userId, true); @@ -93,26 +93,31 @@ public class ParticipantDisplayItem { } } + public boolean isConnected() { + return iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || + iceConnectionState == PeerConnection.IceConnectionState.COMPLETED; + } + + public String getNick() { + if (TextUtils.isEmpty(userId) && TextUtils.isEmpty(nick)) { + return defaultGuestNick; + } + + return nick; + } + + public String getUrlForAvatar() { + return urlForAvatar; + } + public MediaStream getMediaStream() { return mediaStream; } - public void setMediaStream(MediaStream mediaStream) { - this.mediaStream = mediaStream; - - participantDisplayItemNotifier.notifyChange(); - } - public boolean isStreamEnabled() { return streamEnabled; } - public void setStreamEnabled(boolean streamEnabled) { - this.streamEnabled = streamEnabled; - - participantDisplayItemNotifier.notifyChange(); - } - public EglBase getRootEglBase() { return rootEglBase; } @@ -121,12 +126,6 @@ public class ParticipantDisplayItem { return isAudioEnabled; } - public void setAudioEnabled(boolean audioEnabled) { - isAudioEnabled = audioEnabled; - - participantDisplayItemNotifier.notifyChange(); - } - public void addObserver(Observer observer) { participantDisplayItemNotifier.addObserver(observer); } From 4aef76e3478c05c69edc3d25c201d1dec7f5abed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Sat, 26 Nov 2022 04:03:07 +0100 Subject: [PATCH 08/19] Keep track of the stream in the peer connection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../nextcloud/talk/webrtc/PeerConnectionWrapper.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java index 98ef3ba5d..d040d033c 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java @@ -121,6 +121,9 @@ public class PeerConnectionWrapper { private final boolean isMCUPublisher; private final String videoStreamType; + // It is assumed that there will be at most one remote stream at each time. + private MediaStream stream; + @Inject Context context; @@ -219,6 +222,10 @@ public class PeerConnectionWrapper { return videoStreamType; } + public MediaStream getStream() { + return stream; + } + public void removePeerConnection() { signalingMessageReceiver.removeListener(webRtcMessageListener); @@ -484,11 +491,15 @@ public class PeerConnectionWrapper { @Override public void onAddStream(MediaStream mediaStream) { + stream = mediaStream; + peerConnectionNotifier.notifyStreamAdded(mediaStream); } @Override public void onRemoveStream(MediaStream mediaStream) { + stream = null; + peerConnectionNotifier.notifyStreamRemoved(mediaStream); } From 175944e9323a3c2ac0dd4a70700539daf17fdfd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Sat, 26 Nov 2022 04:18:09 +0100 Subject: [PATCH 09/19] Move handling of call participants to its own class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CallParticipant provides a read-only CallParticipantModel and internally handles the data channel and peer connection events that modify the model. Nevertheless, the CallParticipant requires certain properties to be externally set, like the userId or the peer connections. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 133 +++--------- .../nextcloud/talk/call/CallParticipant.java | 198 ++++++++++++++++++ 2 files changed, 224 insertions(+), 107 deletions(-) create mode 100644 app/src/main/java/com/nextcloud/talk/call/CallParticipant.java diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 321b8257b..ab5a79384 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -60,8 +60,8 @@ import com.nextcloud.talk.adapters.ParticipantDisplayItem; import com.nextcloud.talk.adapters.ParticipantsAdapter; import com.nextcloud.talk.api.NcApi; import com.nextcloud.talk.application.NextcloudTalkApplication; +import com.nextcloud.talk.call.CallParticipant; import com.nextcloud.talk.call.CallParticipantModel; -import com.nextcloud.talk.call.MutableCallParticipantModel; import com.nextcloud.talk.data.user.model.User; import com.nextcloud.talk.databinding.CallActivityBinding; import com.nextcloud.talk.events.ConfigurationChangeEvent; @@ -264,11 +264,9 @@ public class CallActivity extends CallBaseActivity { private Map callParticipantMessageListeners = new HashMap<>(); - private Map dataChannelMessageListeners = new HashMap<>(); - private Map peerConnectionObservers = new HashMap<>(); - private Map callParticipantModels = new HashMap<>(); + private Map callParticipants = new HashMap<>(); private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener = new SignalingMessageReceiver.ParticipantListMessageListener() { @@ -385,7 +383,7 @@ public class CallActivity extends CallBaseActivity { requestBluetoothPermission(); } basicInitialization(); - callParticipantModels = new HashMap<>(); + callParticipants = new HashMap<>(); participantDisplayItems = new HashMap<>(); initViews(); if (!isConnectionEstablished()) { @@ -1860,7 +1858,7 @@ public class CallActivity extends CallBaseActivity { String userId = userIdsBySessionId.get(sessionId); if (userId != null) { - callParticipantModels.get(sessionId).setUserId(userId); + callParticipants.get(sessionId).setUserId(userId); } String nick; @@ -1869,7 +1867,7 @@ public class CallActivity extends CallBaseActivity { } else { nick = offerAnswerNickProviders.get(sessionId) != null ? offerAnswerNickProviders.get(sessionId).getNick() : ""; } - callParticipantModels.get(sessionId).setNick(nick); + callParticipants.get(sessionId).setNick(nick); } if (newSessions.size() > 0 && currentCallStatus != CallStatus.IN_CONVERSATION) { @@ -1975,12 +1973,6 @@ public class CallActivity extends CallBaseActivity { new CallActivityCallParticipantMessageListener(sessionId); callParticipantMessageListeners.put(sessionId, callParticipantMessageListener); signalingMessageReceiver.addListener(callParticipantMessageListener, sessionId); - - // DataChannel messages are sent only in video peers; (sender) screen peers do not even open them. - PeerConnectionWrapper.DataChannelMessageListener dataChannelMessageListener = - new CallActivityDataChannelMessageListener(sessionId); - dataChannelMessageListeners.put(sessionId, dataChannelMessageListener); - peerConnectionWrapper.addListener(dataChannelMessageListener); } if (!publisher && !hasExternalSignalingServer && offerAnswerNickProviders.get(sessionId) == null) { @@ -1996,13 +1988,19 @@ public class CallActivity extends CallBaseActivity { peerConnectionWrapper.addObserver(peerConnectionObserver); if (!publisher) { - MutableCallParticipantModel mutableCallParticipantModel = callParticipantModels.get(sessionId); - if (mutableCallParticipantModel == null) { - mutableCallParticipantModel = new MutableCallParticipantModel(sessionId); - callParticipantModels.put(sessionId, mutableCallParticipantModel); + CallParticipant callParticipant = callParticipants.get(sessionId); + if (callParticipant == null) { + callParticipant = new CallParticipant(sessionId); + callParticipants.put(sessionId, callParticipant); } - final CallParticipantModel callParticipantModel = mutableCallParticipantModel; + if ("screen".equals(type)) { + callParticipant.setScreenPeerConnectionWrapper(peerConnectionWrapper); + } else { + callParticipant.setPeerConnectionWrapper(peerConnectionWrapper); + } + + final CallParticipantModel callParticipantModel = callParticipant.getCallParticipantModel(); runOnUiThread(() -> { setupVideoStreamForLayout(callParticipantModel, type); @@ -2033,10 +2031,6 @@ public class CallActivity extends CallBaseActivity { if (!(peerConnectionWrappers = getPeerConnectionWrapperListForSessionId(sessionId)).isEmpty()) { for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrappers) { if (peerConnectionWrapper.getSessionId().equals(sessionId)) { - if (!justScreen && VIDEO_STREAM_TYPE_VIDEO.equals(peerConnectionWrapper.getVideoStreamType())) { - PeerConnectionWrapper.DataChannelMessageListener dataChannelMessageListener = dataChannelMessageListeners.remove(sessionId); - peerConnectionWrapper.removeListener(dataChannelMessageListener); - } String videoStreamType = peerConnectionWrapper.getVideoStreamType(); if (VIDEO_STREAM_TYPE_SCREEN.equals(videoStreamType) || !justScreen) { PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = peerConnectionObservers.remove(sessionId + "-" + videoStreamType); @@ -2044,16 +2038,12 @@ public class CallActivity extends CallBaseActivity { runOnUiThread(() -> removeMediaStream(sessionId, videoStreamType)); - MutableCallParticipantModel mutableCallParticipantModel = callParticipantModels.get(sessionId); - if (mutableCallParticipantModel != null) { + CallParticipant callParticipant = callParticipants.get(sessionId); + if (callParticipant != null) { if ("screen".equals(videoStreamType)) { - mutableCallParticipantModel.setScreenMediaStream(null); - mutableCallParticipantModel.setScreenIceConnectionState(null); + callParticipant.setScreenPeerConnectionWrapper(null); } else { - mutableCallParticipantModel.setMediaStream(null); - mutableCallParticipantModel.setIceConnectionState(null); - mutableCallParticipantModel.setAudioAvailable(null); - mutableCallParticipantModel.setVideoAvailable(null); + callParticipant.setPeerConnectionWrapper(null); } } @@ -2073,7 +2063,10 @@ public class CallActivity extends CallBaseActivity { signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener()); } - callParticipantModels.remove(sessionId); + CallParticipant callParticipant = callParticipants.remove(sessionId); + if (callParticipant != null) { + callParticipant.destroy(); + } } } @@ -2529,8 +2522,8 @@ public class CallActivity extends CallBaseActivity { private void onOfferOrAnswer(String nick) { this.nick = nick; - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setNick(nick); + if (callParticipants.get(sessionId) != null) { + callParticipants.get(sessionId).setNick(nick); } } @@ -2561,50 +2554,6 @@ public class CallActivity extends CallBaseActivity { } } - private class CallActivityDataChannelMessageListener implements PeerConnectionWrapper.DataChannelMessageListener { - - private final String sessionId; - - private CallActivityDataChannelMessageListener(String sessionId) { - this.sessionId = sessionId; - } - - @Override - public void onAudioOn() { - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setAudioAvailable(true); - } - } - - @Override - public void onAudioOff() { - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setAudioAvailable(false); - } - } - - @Override - public void onVideoOn() { - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setVideoAvailable(true); - } - } - - @Override - public void onVideoOff() { - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setVideoAvailable(false); - } - } - - @Override - public void onNickChanged(String nick) { - if (callParticipantModels.get(sessionId) != null) { - callParticipantModels.get(sessionId).setNick(nick); - } - } - } - private class CallActivityPeerConnectionObserver implements PeerConnectionWrapper.PeerConnectionObserver { private final String sessionId; @@ -2617,44 +2566,14 @@ public class CallActivity extends CallBaseActivity { @Override public void onStreamAdded(MediaStream mediaStream) { - handleStream(mediaStream); } @Override public void onStreamRemoved(MediaStream mediaStream) { - handleStream(null); - } - - private void handleStream(MediaStream mediaStream) { - if (callParticipantModels.get(sessionId) == null) { - return; - } - - if ("screen".equals(videoStreamType)) { - callParticipantModels.get(sessionId).setScreenMediaStream(mediaStream); - - return; - } - - boolean hasAtLeastOneVideoStream = false; - if (mediaStream != null) { - hasAtLeastOneVideoStream = mediaStream.videoTracks != null && mediaStream.videoTracks.size() > 0; - } - - callParticipantModels.get(sessionId).setMediaStream(mediaStream); - callParticipantModels.get(sessionId).setVideoAvailable(hasAtLeastOneVideoStream); } @Override public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) { - if (callParticipantModels.get(sessionId) != null) { - if ("screen".equals(videoStreamType)) { - callParticipantModels.get(sessionId).setScreenIceConnectionState(iceConnectionState); - } else { - callParticipantModels.get(sessionId).setIceConnectionState(iceConnectionState); - } - } - runOnUiThread(() -> { if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { updateSelfVideoViewIceConnectionState(iceConnectionState); diff --git a/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java b/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java new file mode 100644 index 000000000..3b153f8cb --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/call/CallParticipant.java @@ -0,0 +1,198 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import com.nextcloud.talk.webrtc.PeerConnectionWrapper; + +import org.webrtc.MediaStream; +import org.webrtc.PeerConnection; + +/** + * Model for (remote) call participants. + * + * This class keeps track of the state changes in a call participant and updates its data model as needed. View classes + * are expected to directly use the read-only data model. + */ +public class CallParticipant { + + private final PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = + new PeerConnectionWrapper.PeerConnectionObserver() { + @Override + public void onStreamAdded(MediaStream mediaStream) { + handleStreamChange(mediaStream); + } + + @Override + public void onStreamRemoved(MediaStream mediaStream) { + handleStreamChange(mediaStream); + } + + @Override + public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) { + handleIceConnectionStateChange(iceConnectionState); + } + }; + + private final PeerConnectionWrapper.PeerConnectionObserver screenPeerConnectionObserver = + new PeerConnectionWrapper.PeerConnectionObserver() { + @Override + public void onStreamAdded(MediaStream mediaStream) { + callParticipantModel.setScreenMediaStream(mediaStream); + } + + @Override + public void onStreamRemoved(MediaStream mediaStream) { + callParticipantModel.setScreenMediaStream(null); + } + + @Override + public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) { + callParticipantModel.setScreenIceConnectionState(iceConnectionState); + } + }; + + // DataChannel messages are sent only in video peers; (sender) screen peers do not even open them. + private final PeerConnectionWrapper.DataChannelMessageListener dataChannelMessageListener = + new PeerConnectionWrapper.DataChannelMessageListener() { + @Override + public void onAudioOn() { + callParticipantModel.setAudioAvailable(Boolean.TRUE); + } + + @Override + public void onAudioOff() { + callParticipantModel.setAudioAvailable(Boolean.FALSE); + } + + @Override + public void onVideoOn() { + callParticipantModel.setVideoAvailable(Boolean.TRUE); + } + + @Override + public void onVideoOff() { + callParticipantModel.setVideoAvailable(Boolean.FALSE); + } + + @Override + public void onNickChanged(String nick) { + callParticipantModel.setNick(nick); + } + }; + + private final MutableCallParticipantModel callParticipantModel; + + private PeerConnectionWrapper peerConnectionWrapper; + private PeerConnectionWrapper screenPeerConnectionWrapper; + + public CallParticipant(String sessionId) { + callParticipantModel = new MutableCallParticipantModel(sessionId); + } + + public void destroy() { + if (peerConnectionWrapper != null) { + peerConnectionWrapper.removeObserver(peerConnectionObserver); + peerConnectionWrapper.removeListener(dataChannelMessageListener); + } + if (screenPeerConnectionWrapper != null) { + screenPeerConnectionWrapper.removeObserver(screenPeerConnectionObserver); + } + } + + public CallParticipantModel getCallParticipantModel() { + return callParticipantModel; + } + + public void setUserId(String userId) { + callParticipantModel.setUserId(userId); + } + + public void setNick(String nick) { + callParticipantModel.setNick(nick); + } + + public void setPeerConnectionWrapper(PeerConnectionWrapper peerConnectionWrapper) { + if (this.peerConnectionWrapper != null) { + this.peerConnectionWrapper.removeObserver(peerConnectionObserver); + this.peerConnectionWrapper.removeListener(dataChannelMessageListener); + } + + this.peerConnectionWrapper = peerConnectionWrapper; + + if (this.peerConnectionWrapper == null) { + callParticipantModel.setIceConnectionState(null); + callParticipantModel.setMediaStream(null); + callParticipantModel.setAudioAvailable(null); + callParticipantModel.setVideoAvailable(null); + + return; + } + + handleIceConnectionStateChange(this.peerConnectionWrapper.getPeerConnection().iceConnectionState()); + handleStreamChange(this.peerConnectionWrapper.getStream()); + + this.peerConnectionWrapper.addObserver(peerConnectionObserver); + this.peerConnectionWrapper.addListener(dataChannelMessageListener); + } + + private void handleIceConnectionStateChange(PeerConnection.IceConnectionState iceConnectionState) { + callParticipantModel.setIceConnectionState(iceConnectionState); + + if (iceConnectionState == PeerConnection.IceConnectionState.NEW || + iceConnectionState == PeerConnection.IceConnectionState.CHECKING) { + callParticipantModel.setAudioAvailable(null); + callParticipantModel.setVideoAvailable(null); + } + } + + private void handleStreamChange(MediaStream mediaStream) { + if (mediaStream == null) { + callParticipantModel.setMediaStream(null); + callParticipantModel.setVideoAvailable(Boolean.FALSE); + + return; + } + + boolean hasAtLeastOneVideoStream = mediaStream.videoTracks != null && !mediaStream.videoTracks.isEmpty(); + + callParticipantModel.setMediaStream(mediaStream); + callParticipantModel.setVideoAvailable(hasAtLeastOneVideoStream); + } + + public void setScreenPeerConnectionWrapper(PeerConnectionWrapper screenPeerConnectionWrapper) { + if (this.screenPeerConnectionWrapper != null) { + this.screenPeerConnectionWrapper.removeObserver(screenPeerConnectionObserver); + } + + this.screenPeerConnectionWrapper = screenPeerConnectionWrapper; + + if (this.screenPeerConnectionWrapper == null) { + callParticipantModel.setScreenIceConnectionState(null); + callParticipantModel.setScreenMediaStream(null); + + return; + } + + callParticipantModel.setScreenIceConnectionState(this.screenPeerConnectionWrapper.getPeerConnection().iceConnectionState()); + callParticipantModel.setScreenMediaStream(this.screenPeerConnectionWrapper.getStream()); + + this.screenPeerConnectionWrapper.addObserver(screenPeerConnectionObserver); + } +} From 5681084a14f0b59b82da217a367b546f9bcb6d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 29 Nov 2022 19:19:44 +0100 Subject: [PATCH 10/19] Create and destroy helper listeners based on call participants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The listeners for call participant messages and for the call participant nick provided by offers / answers were created and destroyed based on the peer connections, although they were implicitly associated to a call participant. Now they are explicitly created and destroyed based on its associated call participant. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 46 +++++++++---------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index ab5a79384..93152fdf3 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1966,22 +1966,6 @@ public class CallActivity extends CallBaseActivity { peerConnectionWrapperList.add(peerConnectionWrapper); - // Currently there is no separation between call participants and peer connections, so any video peer - // connection (except the own publisher connection) is treated as a call participant. - if (!publisher && "video".equals(type)) { - SignalingMessageReceiver.CallParticipantMessageListener callParticipantMessageListener = - new CallActivityCallParticipantMessageListener(sessionId); - callParticipantMessageListeners.put(sessionId, callParticipantMessageListener); - signalingMessageReceiver.addListener(callParticipantMessageListener, sessionId); - } - - if (!publisher && !hasExternalSignalingServer && offerAnswerNickProviders.get(sessionId) == null) { - OfferAnswerNickProvider offerAnswerNickProvider = new OfferAnswerNickProvider(sessionId); - offerAnswerNickProviders.put(sessionId, offerAnswerNickProvider); - signalingMessageReceiver.addListener(offerAnswerNickProvider.getVideoWebRtcMessageListener(), sessionId, "video"); - signalingMessageReceiver.addListener(offerAnswerNickProvider.getScreenWebRtcMessageListener(), sessionId, "screen"); - } - PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = new CallActivityPeerConnectionObserver(sessionId, type); peerConnectionObservers.put(sessionId + "-" + type, peerConnectionObserver); @@ -1992,6 +1976,18 @@ public class CallActivity extends CallBaseActivity { if (callParticipant == null) { callParticipant = new CallParticipant(sessionId); callParticipants.put(sessionId, callParticipant); + + SignalingMessageReceiver.CallParticipantMessageListener callParticipantMessageListener = + new CallActivityCallParticipantMessageListener(sessionId); + callParticipantMessageListeners.put(sessionId, callParticipantMessageListener); + signalingMessageReceiver.addListener(callParticipantMessageListener, sessionId); + + if (!hasExternalSignalingServer) { + OfferAnswerNickProvider offerAnswerNickProvider = new OfferAnswerNickProvider(sessionId); + offerAnswerNickProviders.put(sessionId, offerAnswerNickProvider); + signalingMessageReceiver.addListener(offerAnswerNickProvider.getVideoWebRtcMessageListener(), sessionId, "video"); + signalingMessageReceiver.addListener(offerAnswerNickProvider.getScreenWebRtcMessageListener(), sessionId, "screen"); + } } if ("screen".equals(type)) { @@ -2054,18 +2050,18 @@ public class CallActivity extends CallBaseActivity { } if (!justScreen) { - SignalingMessageReceiver.CallParticipantMessageListener listener = callParticipantMessageListeners.remove(sessionId); - signalingMessageReceiver.removeListener(listener); - - OfferAnswerNickProvider offerAnswerNickProvider = offerAnswerNickProviders.remove(sessionId); - if (offerAnswerNickProvider != null) { - signalingMessageReceiver.removeListener(offerAnswerNickProvider.getVideoWebRtcMessageListener()); - signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener()); - } - CallParticipant callParticipant = callParticipants.remove(sessionId); if (callParticipant != null) { callParticipant.destroy(); + + SignalingMessageReceiver.CallParticipantMessageListener listener = callParticipantMessageListeners.remove(sessionId); + signalingMessageReceiver.removeListener(listener); + + OfferAnswerNickProvider offerAnswerNickProvider = offerAnswerNickProviders.remove(sessionId); + if (offerAnswerNickProvider != null) { + signalingMessageReceiver.removeListener(offerAnswerNickProvider.getVideoWebRtcMessageListener()); + signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener()); + } } } } From e17a999812e91ce74bb1c6c77ec345d2279ba312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Sun, 27 Nov 2022 21:58:49 +0100 Subject: [PATCH 11/19] Rename methods to add and remove ParticipantDisplayItems MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../com/nextcloud/talk/activities/CallActivity.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 93152fdf3..029aaf29d 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1999,7 +1999,7 @@ public class CallActivity extends CallBaseActivity { final CallParticipantModel callParticipantModel = callParticipant.getCallParticipantModel(); runOnUiThread(() -> { - setupVideoStreamForLayout(callParticipantModel, type); + addParticipantDisplayItem(callParticipantModel, type); }); } @@ -2032,7 +2032,7 @@ public class CallActivity extends CallBaseActivity { PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = peerConnectionObservers.remove(sessionId + "-" + videoStreamType); peerConnectionWrapper.removeObserver(peerConnectionObserver); - runOnUiThread(() -> removeMediaStream(sessionId, videoStreamType)); + runOnUiThread(() -> removeParticipantDisplayItem(sessionId, videoStreamType)); CallParticipant callParticipant = callParticipants.get(sessionId); if (callParticipant != null) { @@ -2066,8 +2066,8 @@ public class CallActivity extends CallBaseActivity { } } - private void removeMediaStream(String sessionId, String videoStreamType) { - Log.d(TAG, "removeMediaStream"); + private void removeParticipantDisplayItem(String sessionId, String videoStreamType) { + Log.d(TAG, "removeParticipantDisplayItem"); ParticipantDisplayItem participantDisplayItem = participantDisplayItems.remove(sessionId + "-" + videoStreamType); if (participantDisplayItem == null) { return; @@ -2200,7 +2200,7 @@ public class CallActivity extends CallBaseActivity { this); } - private void setupVideoStreamForLayout(CallParticipantModel callParticipantModel, String videoStreamType) { + private void addParticipantDisplayItem(CallParticipantModel callParticipantModel, String videoStreamType) { String defaultGuestNick = getResources().getString(R.string.nc_nick_guest); ParticipantDisplayItem participantDisplayItem = new ParticipantDisplayItem(baseUrl, From 534bbddc88fa986406399c9e9e31975264d93bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Sun, 27 Nov 2022 22:12:46 +0100 Subject: [PATCH 12/19] Create and destroy ParticipantDisplayItems based on call participants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ParticipantDisplayItems were created and destroyed based on the peer connections. Now a ParticipantDisplayItem of "video" type is associated to a call participant, while an additional item is created and destroyed depending on the state of the screen peer connection of the call participant. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 53 ++++++++++++++++--- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 029aaf29d..343111e8d 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -268,6 +268,10 @@ public class CallActivity extends CallBaseActivity { private Map callParticipants = new HashMap<>(); + private Map screenParticipantDisplayItemManagers = new HashMap<>(); + + private Handler screenParticipantDisplayItemManagersHandler = new Handler(Looper.getMainLooper()); + private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener = new SignalingMessageReceiver.ParticipantListMessageListener() { @Override @@ -1988,6 +1992,17 @@ public class CallActivity extends CallBaseActivity { signalingMessageReceiver.addListener(offerAnswerNickProvider.getVideoWebRtcMessageListener(), sessionId, "video"); signalingMessageReceiver.addListener(offerAnswerNickProvider.getScreenWebRtcMessageListener(), sessionId, "screen"); } + + final CallParticipantModel callParticipantModel = callParticipant.getCallParticipantModel(); + + ScreenParticipantDisplayItemManager screenParticipantDisplayItemManager = + new ScreenParticipantDisplayItemManager(callParticipantModel); + screenParticipantDisplayItemManagers.put(sessionId, screenParticipantDisplayItemManager); + callParticipantModel.addObserver(screenParticipantDisplayItemManager, screenParticipantDisplayItemManagersHandler); + + runOnUiThread(() -> { + addParticipantDisplayItem(callParticipantModel, "video"); + }); } if ("screen".equals(type)) { @@ -1995,12 +2010,6 @@ public class CallActivity extends CallBaseActivity { } else { callParticipant.setPeerConnectionWrapper(peerConnectionWrapper); } - - final CallParticipantModel callParticipantModel = callParticipant.getCallParticipantModel(); - - runOnUiThread(() -> { - addParticipantDisplayItem(callParticipantModel, type); - }); } if (publisher) { @@ -2032,8 +2041,6 @@ public class CallActivity extends CallBaseActivity { PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = peerConnectionObservers.remove(sessionId + "-" + videoStreamType); peerConnectionWrapper.removeObserver(peerConnectionObserver); - runOnUiThread(() -> removeParticipantDisplayItem(sessionId, videoStreamType)); - CallParticipant callParticipant = callParticipants.get(sessionId); if (callParticipant != null) { if ("screen".equals(videoStreamType)) { @@ -2052,6 +2059,10 @@ public class CallActivity extends CallBaseActivity { if (!justScreen) { CallParticipant callParticipant = callParticipants.remove(sessionId); if (callParticipant != null) { + ScreenParticipantDisplayItemManager screenParticipantDisplayItemManager = + screenParticipantDisplayItemManagers.remove(sessionId); + callParticipant.getCallParticipantModel().removeObserver(screenParticipantDisplayItemManager); + callParticipant.destroy(); SignalingMessageReceiver.CallParticipantMessageListener listener = callParticipantMessageListeners.remove(sessionId); @@ -2062,6 +2073,8 @@ public class CallActivity extends CallBaseActivity { signalingMessageReceiver.removeListener(offerAnswerNickProvider.getVideoWebRtcMessageListener()); signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener()); } + + runOnUiThread(() -> removeParticipantDisplayItem(sessionId, "video")); } } } @@ -2594,6 +2607,30 @@ public class CallActivity extends CallBaseActivity { } } + private class ScreenParticipantDisplayItemManager implements CallParticipantModel.Observer { + + private final CallParticipantModel callParticipantModel; + + private ScreenParticipantDisplayItemManager(CallParticipantModel callParticipantModel) { + this.callParticipantModel = callParticipantModel; + } + + @Override + public void onChange() { + String sessionId = callParticipantModel.getSessionId(); + if (callParticipantModel.getScreenIceConnectionState() == null) { + removeParticipantDisplayItem(sessionId, "screen"); + + return; + } + + boolean hasScreenParticipantDisplayItem = participantDisplayItems.get(sessionId + "-screen") != null; + if (!hasScreenParticipantDisplayItem) { + addParticipantDisplayItem(callParticipantModel, "screen"); + } + } + } + private class InternalSignalingMessageSender implements SignalingMessageSender { @Override From 6728e3f063b18cdc173f930267dbc83e4191f85b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Mon, 28 Nov 2022 09:12:27 +0100 Subject: [PATCH 13/19] Do not handle connection state changes to "closed" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The connection state changes to "closed" only when the connection is closed. However, closing a connection does not fire any event (not even the "iceConnectionStateChanged" event), so the event handler can be removed as it will never be executed. Signed-off-by: Daniel Calviño Sánchez --- .../com/nextcloud/talk/activities/CallActivity.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 343111e8d..f342be713 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1971,7 +1971,7 @@ public class CallActivity extends CallBaseActivity { peerConnectionWrapperList.add(peerConnectionWrapper); PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = - new CallActivityPeerConnectionObserver(sessionId, type); + new CallActivityPeerConnectionObserver(sessionId); peerConnectionObservers.put(sessionId + "-" + type, peerConnectionObserver); peerConnectionWrapper.addObserver(peerConnectionObserver); @@ -2566,11 +2566,9 @@ public class CallActivity extends CallBaseActivity { private class CallActivityPeerConnectionObserver implements PeerConnectionWrapper.PeerConnectionObserver { private final String sessionId; - private final String videoStreamType; - private CallActivityPeerConnectionObserver(String sessionId, String videoStreamType) { + private CallActivityPeerConnectionObserver(String sessionId) { this.sessionId = sessionId; - this.videoStreamType = videoStreamType; } @Override @@ -2588,12 +2586,6 @@ public class CallActivity extends CallBaseActivity { updateSelfVideoViewIceConnectionState(iceConnectionState); } - if (iceConnectionState == PeerConnection.IceConnectionState.CLOSED) { - endPeerConnection(sessionId, VIDEO_STREAM_TYPE_SCREEN.equals(videoStreamType)); - - return; - } - if (iceConnectionState == PeerConnection.IceConnectionState.FAILED) { if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { setCallState(CallStatus.PUBLISHER_FAILED); From 0a3f515bb6e1bfd5475177f716b53590577e46db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Mon, 28 Nov 2022 10:01:33 +0100 Subject: [PATCH 14/19] Observe only the self peer connection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The observers were created for any peer connection, but after recent changes they ignored all changes but those from the self peer connection. Therefore it is enough to just add an explicit listener on that peer connection rather than on all of them. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 38 ++++++------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index f342be713..df89f8b0b 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -264,7 +264,7 @@ public class CallActivity extends CallBaseActivity { private Map callParticipantMessageListeners = new HashMap<>(); - private Map peerConnectionObservers = new HashMap<>(); + private PeerConnectionWrapper.PeerConnectionObserver selfPeerConnectionObserver = new CallActivitySelfPeerConnectionObserver(); private Map callParticipants = new HashMap<>(); @@ -1970,11 +1970,6 @@ public class CallActivity extends CallBaseActivity { peerConnectionWrapperList.add(peerConnectionWrapper); - PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = - new CallActivityPeerConnectionObserver(sessionId); - peerConnectionObservers.put(sessionId + "-" + type, peerConnectionObserver); - peerConnectionWrapper.addObserver(peerConnectionObserver); - if (!publisher) { CallParticipant callParticipant = callParticipants.get(sessionId); if (callParticipant == null) { @@ -2013,6 +2008,8 @@ public class CallActivity extends CallBaseActivity { } if (publisher) { + peerConnectionWrapper.addObserver(selfPeerConnectionObserver); + startSendingNick(); } @@ -2036,11 +2033,12 @@ public class CallActivity extends CallBaseActivity { if (!(peerConnectionWrappers = getPeerConnectionWrapperListForSessionId(sessionId)).isEmpty()) { for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrappers) { if (peerConnectionWrapper.getSessionId().equals(sessionId)) { + if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { + peerConnectionWrapper.removeObserver(selfPeerConnectionObserver); + } + String videoStreamType = peerConnectionWrapper.getVideoStreamType(); if (VIDEO_STREAM_TYPE_SCREEN.equals(videoStreamType) || !justScreen) { - PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver = peerConnectionObservers.remove(sessionId + "-" + videoStreamType); - peerConnectionWrapper.removeObserver(peerConnectionObserver); - CallParticipant callParticipant = callParticipants.get(sessionId); if (callParticipant != null) { if ("screen".equals(videoStreamType)) { @@ -2563,13 +2561,7 @@ public class CallActivity extends CallBaseActivity { } } - private class CallActivityPeerConnectionObserver implements PeerConnectionWrapper.PeerConnectionObserver { - - private final String sessionId; - - private CallActivityPeerConnectionObserver(String sessionId) { - this.sessionId = sessionId; - } + private class CallActivitySelfPeerConnectionObserver implements PeerConnectionWrapper.PeerConnectionObserver { @Override public void onStreamAdded(MediaStream mediaStream) { @@ -2582,18 +2574,12 @@ public class CallActivity extends CallBaseActivity { @Override public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) { runOnUiThread(() -> { - if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { - updateSelfVideoViewIceConnectionState(iceConnectionState); - } + updateSelfVideoViewIceConnectionState(iceConnectionState); if (iceConnectionState == PeerConnection.IceConnectionState.FAILED) { - if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { - setCallState(CallStatus.PUBLISHER_FAILED); - webSocketClient.clearResumeId(); - hangup(false); - } - - return; + setCallState(CallStatus.PUBLISHER_FAILED); + webSocketClient.clearResumeId(); + hangup(false); } }); } From ab72db7a10538f12c0213904f1527f8c5058a71c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 29 Nov 2022 13:04:17 +0100 Subject: [PATCH 15/19] Add helper class to keep track of the participants in a call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For now only the same signaling messages that were already handled are still handled; in the future it could be extended to handle other messages, like the one sent by the external signaling server when a participant leaves the room (in some cases no participants update message is sent if the participant leaves the call and room at the same time, which causes the participants to still be seen as in call until a new update is received). Signed-off-by: Daniel Calviño Sánchez --- .../talk/call/CallParticipantList.java | 164 +++++ .../call/CallParticipantListNotifier.java | 63 ++ ...lParticipantListExternalSignalingTest.java | 663 ++++++++++++++++++ ...lParticipantListInternalSignalingTest.java | 535 ++++++++++++++ .../talk/call/CallParticipantListTest.java | 61 ++ 5 files changed, 1486 insertions(+) create mode 100644 app/src/main/java/com/nextcloud/talk/call/CallParticipantList.java create mode 100644 app/src/main/java/com/nextcloud/talk/call/CallParticipantListNotifier.java create mode 100644 app/src/test/java/com/nextcloud/talk/call/CallParticipantListExternalSignalingTest.java create mode 100644 app/src/test/java/com/nextcloud/talk/call/CallParticipantListInternalSignalingTest.java create mode 100644 app/src/test/java/com/nextcloud/talk/call/CallParticipantListTest.java diff --git a/app/src/main/java/com/nextcloud/talk/call/CallParticipantList.java b/app/src/main/java/com/nextcloud/talk/call/CallParticipantList.java new file mode 100644 index 000000000..6135ab991 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/call/CallParticipantList.java @@ -0,0 +1,164 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import com.nextcloud.talk.models.json.participants.Participant; +import com.nextcloud.talk.signaling.SignalingMessageReceiver; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Helper class to keep track of the participants in a call based on the signaling messages. + * + * The CallParticipantList adds a listener for participant list messages as soon as it is created and starts tracking + * the call participants until destroyed. Notifications about the changes can be received by adding an observer to the + * CallParticipantList; note that no sorting is guaranteed on the participants. + */ +public class CallParticipantList { + + public interface Observer { + void onCallParticipantsChanged(Collection joined, Collection updated, + Collection left, Collection unchanged); + void onCallEndedForAll(); + } + + private final SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener = + new SignalingMessageReceiver.ParticipantListMessageListener() { + + private final Map callParticipants = new HashMap<>(); + + @Override + public void onUsersInRoom(List participants) { + processParticipantList(participants); + } + + @Override + public void onParticipantsUpdate(List participants) { + processParticipantList(participants); + } + + private void processParticipantList(List participants) { + Collection joined = new ArrayList<>(); + Collection updated = new ArrayList<>(); + Collection left = new ArrayList<>(); + Collection unchanged = new ArrayList<>(); + + Collection knownCallParticipantsNotFound = new ArrayList<>(callParticipants.values()); + + for (Participant participant : participants) { + String sessionId = participant.getSessionId(); + Participant callParticipant = callParticipants.get(sessionId); + + boolean knownCallParticipant = callParticipant != null; + if (!knownCallParticipant && participant.getInCall() != Participant.InCallFlags.DISCONNECTED) { + callParticipants.put(sessionId, copyParticipant(participant)); + joined.add(copyParticipant(participant)); + } else if (knownCallParticipant && participant.getInCall() == Participant.InCallFlags.DISCONNECTED) { + callParticipants.remove(sessionId); + // No need to copy it, as it will be no longer used. + callParticipant.setInCall(Participant.InCallFlags.DISCONNECTED); + left.add(callParticipant); + } else if (knownCallParticipant && callParticipant.getInCall() != participant.getInCall()) { + callParticipant.setInCall(participant.getInCall()); + updated.add(copyParticipant(participant)); + } else if (knownCallParticipant) { + unchanged.add(copyParticipant(participant)); + } + + if (knownCallParticipant) { + knownCallParticipantsNotFound.remove(callParticipant); + } + } + + for (Participant callParticipant : knownCallParticipantsNotFound) { + callParticipants.remove(callParticipant.getSessionId()); + // No need to copy it, as it will be no longer used. + callParticipant.setInCall(Participant.InCallFlags.DISCONNECTED); + left.add(callParticipant); + } + + if (!joined.isEmpty() || !updated.isEmpty() || !left.isEmpty()) { + callParticipantListNotifier.notifyChanged(joined, updated, left, unchanged); + } + } + + @Override + public void onAllParticipantsUpdate(long inCall) { + if (inCall != Participant.InCallFlags.DISCONNECTED) { + // Updating all participants is expected to happen only to disconnect them. + return; + } + + callParticipantListNotifier.notifyCallEndedForAll(); + + Collection joined = new ArrayList<>(); + Collection updated = new ArrayList<>(); + Collection left = new ArrayList<>(callParticipants.size()); + Collection unchanged = new ArrayList<>(); + + for (Participant callParticipant : callParticipants.values()) { + // No need to copy it, as it will be no longer used. + callParticipant.setInCall(Participant.InCallFlags.DISCONNECTED); + left.add(callParticipant); + } + callParticipants.clear(); + + if (!left.isEmpty()) { + callParticipantListNotifier.notifyChanged(joined, updated, left, unchanged); + } + } + + private Participant copyParticipant(Participant participant) { + Participant copiedParticipant = new Participant(); + copiedParticipant.setInCall(participant.getInCall()); + copiedParticipant.setLastPing(participant.getLastPing()); + copiedParticipant.setSessionId(participant.getSessionId()); + copiedParticipant.setType(participant.getType()); + copiedParticipant.setUserId(participant.getUserId()); + + return copiedParticipant; + } + }; + + private final CallParticipantListNotifier callParticipantListNotifier = new CallParticipantListNotifier(); + + private final SignalingMessageReceiver signalingMessageReceiver; + + public CallParticipantList(SignalingMessageReceiver signalingMessageReceiver) { + this.signalingMessageReceiver = signalingMessageReceiver; + this.signalingMessageReceiver.addListener(participantListMessageListener); + } + + public void destroy() { + signalingMessageReceiver.removeListener(participantListMessageListener); + } + + public void addObserver(Observer observer) { + callParticipantListNotifier.addObserver(observer); + } + + public void removeObserver(Observer observer) { + callParticipantListNotifier.removeObserver(observer); + } +} diff --git a/app/src/main/java/com/nextcloud/talk/call/CallParticipantListNotifier.java b/app/src/main/java/com/nextcloud/talk/call/CallParticipantListNotifier.java new file mode 100644 index 000000000..afbc893fd --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/call/CallParticipantListNotifier.java @@ -0,0 +1,63 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import com.nextcloud.talk.models.json.participants.Participant; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Helper class to register and notify CallParticipantList.Observers. + * + * This class is only meant for internal use by CallParticipantList; listeners must register themselves against + * a CallParticipantList rather than against a CallParticipantListNotifier. + */ +class CallParticipantListNotifier { + + private final Set callParticipantListObservers = new LinkedHashSet<>(); + + public synchronized void addObserver(CallParticipantList.Observer observer) { + if (observer == null) { + throw new IllegalArgumentException("CallParticipantList.Observer can not be null"); + } + + callParticipantListObservers.add(observer); + } + + public synchronized void removeObserver(CallParticipantList.Observer observer) { + callParticipantListObservers.remove(observer); + } + + public synchronized void notifyChanged(Collection joined, Collection updated, + Collection left, Collection unchanged) { + for (CallParticipantList.Observer observer : new ArrayList<>(callParticipantListObservers)) { + observer.onCallParticipantsChanged(joined, updated, left, unchanged); + } + } + + public synchronized void notifyCallEndedForAll() { + for (CallParticipantList.Observer observer : new ArrayList<>(callParticipantListObservers)) { + observer.onCallEndedForAll(); + } + } +} diff --git a/app/src/test/java/com/nextcloud/talk/call/CallParticipantListExternalSignalingTest.java b/app/src/test/java/com/nextcloud/talk/call/CallParticipantListExternalSignalingTest.java new file mode 100644 index 000000000..7d2012207 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/call/CallParticipantListExternalSignalingTest.java @@ -0,0 +1,663 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import com.nextcloud.talk.models.json.participants.Participant; +import com.nextcloud.talk.signaling.SignalingMessageReceiver; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatcher; +import org.mockito.InOrder; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.DISCONNECTED; +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.IN_CALL; +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.WITH_AUDIO; +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.WITH_VIDEO; +import static com.nextcloud.talk.models.json.participants.Participant.ParticipantType.GUEST; +import static com.nextcloud.talk.models.json.participants.Participant.ParticipantType.GUEST_MODERATOR; +import static com.nextcloud.talk.models.json.participants.Participant.ParticipantType.MODERATOR; +import static com.nextcloud.talk.models.json.participants.Participant.ParticipantType.OWNER; +import static com.nextcloud.talk.models.json.participants.Participant.ParticipantType.USER; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.only; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +public class CallParticipantListExternalSignalingTest { + + private static class ParticipantsUpdateParticipantBuilder { + private Participant newUser(long inCall, long lastPing, String sessionId, Participant.ParticipantType type, + String userId) { + Participant participant = new Participant(); + participant.setInCall(inCall); + participant.setLastPing(lastPing); + participant.setSessionId(sessionId); + participant.setType(type); + participant.setUserId(userId); + + return participant; + } + + private Participant newGuest(long inCall, long lastPing, String sessionId, Participant.ParticipantType type) { + Participant participant = new Participant(); + participant.setInCall(inCall); + participant.setLastPing(lastPing); + participant.setSessionId(sessionId); + participant.setType(type); + + return participant; + } + } + + private final ParticipantsUpdateParticipantBuilder builder = new ParticipantsUpdateParticipantBuilder(); + + private CallParticipantList callParticipantList; + private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener; + + private CallParticipantList.Observer mockedCallParticipantListObserver; + + private Collection expectedJoined; + private Collection expectedUpdated; + private Collection expectedLeft; + private Collection expectedUnchanged; + + // The order of the left participants in some tests depends on how they are internally sorted by the map, so the + // list of left participants needs to be checked ignoring the sorting (or, rather, sorting by session ID as in + // expectedLeft). + // Other tests can just relay on the not guaranteed, but known internal sorting of the elements. + private final ArgumentMatcher> matchesExpectedLeftIgnoringOrder = left -> { + Collections.sort(left, Comparator.comparing(Participant::getSessionId)); + return expectedLeft.equals(left); + }; + + @Before + public void setUp() { + SignalingMessageReceiver mockedSignalingMessageReceiver = mock(SignalingMessageReceiver.class); + + callParticipantList = new CallParticipantList(mockedSignalingMessageReceiver); + + mockedCallParticipantListObserver = mock(CallParticipantList.Observer.class); + + // Get internal ParticipantListMessageListener from callParticipantList set in the + // mockedSignalingMessageReceiver. + ArgumentCaptor participantListMessageListenerArgumentCaptor = + ArgumentCaptor.forClass(SignalingMessageReceiver.ParticipantListMessageListener.class); + + verify(mockedSignalingMessageReceiver).addListener(participantListMessageListenerArgumentCaptor.capture()); + + participantListMessageListener = participantListMessageListenerArgumentCaptor.getValue(); + + expectedJoined = new ArrayList<>(); + expectedUpdated = new ArrayList<>(); + expectedLeft = new ArrayList<>(); + expectedUnchanged = new ArrayList<>(); + } + + @Test + public void testParticipantsUpdateJoinRoom() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateJoinRoomSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(DISCONNECTED, 4, "theSessionId4", USER, "theUserId4")); + participants.add(builder.newUser(DISCONNECTED, 5, "theSessionId5", OWNER, "theUserId5")); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateJoinRoomThenJoinCall() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateJoinRoomThenJoinCallSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(DISCONNECTED, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + expectedJoined.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateJoinRoomAndCall() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateJoinRoomAndCallSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + expectedJoined.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateJoinRoomAndCallRepeated() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + participantListMessageListener.onParticipantsUpdate(participants); + participantListMessageListener.onParticipantsUpdate(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateChangeCallFlags() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedUpdated.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateChangeCallFlagsSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO | WITH_VIDEO, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO | WITH_VIDEO, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL | WITH_VIDEO, 4, "theSessionId4", USER, "theUserId4")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedUpdated.add(builder.newUser(IN_CALL, 1, "theSessionId1", MODERATOR, "theUserId1")); + expectedUpdated.add(builder.newUser(IN_CALL | WITH_VIDEO, 4, "theSessionId4", USER, "theUserId4")); + expectedUnchanged.add(builder.newGuest(IN_CALL | WITH_AUDIO | WITH_VIDEO, 2, "theSessionId2", GUEST)); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateChangeLastPing() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId1", MODERATOR, "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateChangeLastPingSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 3, "theSessionId3", USER, "theUserId3")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 108, "theSessionId2", GUEST)); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 815, "theSessionId3", USER, "theUserId3")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateChangeParticipantType() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", USER, "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateChangeParticipantTypeeSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 3, "theSessionId3", USER, "theUserId3")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", USER, "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 2, "theSessionId2", GUEST_MODERATOR)); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 3, "theSessionId3", MODERATOR, "theUserId3")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateLeaveCall() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateLeaveCallSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + expectedLeft.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateLeaveCallThenLeaveRoom() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateLeaveCallThenLeaveRoomSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testParticipantsUpdateLeaveCallAndRoom() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testParticipantsUpdateLeaveCallAndRoomSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + expectedLeft.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + + verify(mockedCallParticipantListObserver).onCallParticipantsChanged(eq(expectedJoined), eq(expectedUpdated), + argThat(matchesExpectedLeftIgnoringOrder), eq(expectedUnchanged)); + } + + @Test + public void testParticipantsUpdateSeveralEventsSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + participants.add(builder.newUser(IN_CALL, 5, "theSessionId5", OWNER, "theUserId5")); + // theSessionId6 has not joined yet. + participants.add(builder.newGuest(IN_CALL | WITH_VIDEO, 7, "theSessionId7", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 8, "theSessionId8", USER, "theUserId8")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 9, "theSessionId9", MODERATOR, "theUserId9")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + // theSessionId1 is gone. + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 5, "theSessionId5", OWNER, "theUserId5")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 6, "theSessionId6", GUEST)); + participants.add(builder.newGuest(IN_CALL, 7, "theSessionId7", GUEST)); + participants.add(builder.newUser(IN_CALL, 8, "theSessionId8", USER, "theUserId8")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId9", USER, "theUserId9")); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedJoined.add(builder.newGuest(IN_CALL | WITH_AUDIO, 6, "theSessionId6", GUEST)); + expectedJoined.add(builder.newUser(IN_CALL, 8, "theSessionId8", USER, "theUserId8")); + expectedUpdated.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 5, "theSessionId5", OWNER, "theUserId5")); + expectedUpdated.add(builder.newGuest(IN_CALL, 7, "theSessionId7", GUEST)); + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + expectedLeft.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2", GUEST)); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", USER, "theUserId4")); + // Last ping and participant type are not seen as changed, even if they did. + expectedUnchanged.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId9", USER, "theUserId9")); + + verify(mockedCallParticipantListObserver).onCallParticipantsChanged(eq(expectedJoined), eq(expectedUpdated), + argThat(matchesExpectedLeftIgnoringOrder), eq(expectedUnchanged)); + } + + @Test + public void testAllParticipantsUpdateDisconnected() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onAllParticipantsUpdate(DISCONNECTED); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + + InOrder inOrder = inOrder(mockedCallParticipantListObserver); + + inOrder.verify(mockedCallParticipantListObserver).onCallEndedForAll(); + inOrder.verify(mockedCallParticipantListObserver).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testAllParticipantsUpdateDisconnectedWithSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL, 1, "theSessionId1", MODERATOR, "theUserId1")); + participants.add(builder.newUser(DISCONNECTED, 2, "theSessionId2", USER, "theUserId2")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 3, "theSessionId3", USER, "theUserId3")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO | WITH_VIDEO, 4, "theSessionId4", GUEST)); + + participantListMessageListener.onParticipantsUpdate(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onAllParticipantsUpdate(DISCONNECTED); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", MODERATOR, "theUserId1")); + expectedLeft.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", USER, "theUserId3")); + expectedLeft.add(builder.newGuest(DISCONNECTED, 4, "theSessionId4", GUEST)); + + InOrder inOrder = inOrder(mockedCallParticipantListObserver); + + inOrder.verify(mockedCallParticipantListObserver).onCallEndedForAll(); + inOrder.verify(mockedCallParticipantListObserver).onCallParticipantsChanged(eq(expectedJoined), eq(expectedUpdated), + argThat(matchesExpectedLeftIgnoringOrder), eq(expectedUnchanged)); + } + + @Test + public void testAllParticipantsUpdateDisconnectedNoOneInCall() { + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onAllParticipantsUpdate(DISCONNECTED); + + InOrder inOrder = inOrder(mockedCallParticipantListObserver); + + inOrder.verify(mockedCallParticipantListObserver).onCallEndedForAll(); + verifyNoMoreInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testAllParticipantsUpdateDisconnectedThenJoinCallAgain() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + participantListMessageListener.onAllParticipantsUpdate(DISCONNECTED); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL, 1, "theSessionId1", MODERATOR, "theUserId1")); + + participantListMessageListener.onParticipantsUpdate(participants); + + expectedJoined.add(builder.newUser(IN_CALL, 1, "theSessionId1", MODERATOR, "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } +} diff --git a/app/src/test/java/com/nextcloud/talk/call/CallParticipantListInternalSignalingTest.java b/app/src/test/java/com/nextcloud/talk/call/CallParticipantListInternalSignalingTest.java new file mode 100644 index 000000000..fb393b10f --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/call/CallParticipantListInternalSignalingTest.java @@ -0,0 +1,535 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import com.nextcloud.talk.models.json.participants.Participant; +import com.nextcloud.talk.signaling.SignalingMessageReceiver; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatcher; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.DISCONNECTED; +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.IN_CALL; +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.WITH_AUDIO; +import static com.nextcloud.talk.models.json.participants.Participant.InCallFlags.WITH_VIDEO; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.only; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +public class CallParticipantListInternalSignalingTest { + + private static class UsersInRoomParticipantBuilder { + private Participant newUser(long inCall, long lastPing, String sessionId, String userId) { + Participant participant = new Participant(); + participant.setInCall(inCall); + participant.setLastPing(lastPing); + participant.setSessionId(sessionId); + participant.setUserId(userId); + + return participant; + } + + private Participant newGuest(long inCall, long lastPing, String sessionId) { + Participant participant = new Participant(); + participant.setInCall(inCall); + participant.setLastPing(lastPing); + participant.setSessionId(sessionId); + + return participant; + } + } + + private final UsersInRoomParticipantBuilder builder = new UsersInRoomParticipantBuilder(); + + private CallParticipantList callParticipantList; + private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener; + + private CallParticipantList.Observer mockedCallParticipantListObserver; + + private Collection expectedJoined; + private Collection expectedUpdated; + private Collection expectedLeft; + private Collection expectedUnchanged; + + // The order of the left participants in some tests depends on how they are internally sorted by the map, so the + // list of left participants needs to be checked ignoring the sorting (or, rather, sorting by session ID as in + // expectedLeft). + // Other tests can just relay on the not guaranteed, but known internal sorting of the elements. + private final ArgumentMatcher> matchesExpectedLeftIgnoringOrder = left -> { + Collections.sort(left, Comparator.comparing(Participant::getSessionId)); + return expectedLeft.equals(left); + }; + + @Before + public void setUp() { + SignalingMessageReceiver mockedSignalingMessageReceiver = mock(SignalingMessageReceiver.class); + + callParticipantList = new CallParticipantList(mockedSignalingMessageReceiver); + + mockedCallParticipantListObserver = mock(CallParticipantList.Observer.class); + + // Get internal ParticipantListMessageListener from callParticipantList set in the + // mockedSignalingMessageReceiver. + ArgumentCaptor participantListMessageListenerArgumentCaptor = + ArgumentCaptor.forClass(SignalingMessageReceiver.ParticipantListMessageListener.class); + + verify(mockedSignalingMessageReceiver).addListener(participantListMessageListenerArgumentCaptor.capture()); + + participantListMessageListener = participantListMessageListenerArgumentCaptor.getValue(); + + expectedJoined = new ArrayList<>(); + expectedUpdated = new ArrayList<>(); + expectedLeft = new ArrayList<>(); + expectedUnchanged = new ArrayList<>(); + } + + @Test + public void testUsersInRoomJoinRoom() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testUsersInRoomJoinRoomSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(DISCONNECTED, 4, "theSessionId4", "theUserId4")); + participants.add(builder.newUser(DISCONNECTED, 5, "theSessionId5", "theUserId5")); + + participantListMessageListener.onUsersInRoom(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testUsersInRoomJoinRoomThenJoinCall() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomJoinRoomThenJoinCallSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(DISCONNECTED, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + expectedJoined.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomJoinRoomAndCall() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomJoinRoomAndCallSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + expectedJoined.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomJoinRoomAndCallRepeated() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + participantListMessageListener.onUsersInRoom(participants); + participantListMessageListener.onUsersInRoom(participants); + + expectedJoined.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomChangeCallFlags() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 1, "theSessionId1", "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + expectedUpdated.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 1, "theSessionId1", "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomChangeCallFlagsSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO | WITH_VIDEO, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO | WITH_VIDEO, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL | WITH_VIDEO, 4, "theSessionId4", "theUserId4")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + expectedUpdated.add(builder.newUser(IN_CALL, 1, "theSessionId1", "theUserId1")); + expectedUpdated.add(builder.newUser(IN_CALL | WITH_VIDEO, 4, "theSessionId4", "theUserId4")); + expectedUnchanged.add(builder.newGuest(IN_CALL | WITH_AUDIO | WITH_VIDEO, 2, "theSessionId2")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomChangeLastPing() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId1", "theUserId1")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testUsersInRoomChangeLastPingSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 2, "theSessionId2")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 3, "theSessionId3", "theUserId3")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 108, "theSessionId2")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 815, "theSessionId3", "theUserId3")); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participantListMessageListener.onUsersInRoom(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testUsersInRoomLeaveCall() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomLeaveCallSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + expectedLeft.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomLeaveCallThenLeaveRoom() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + + participantListMessageListener.onUsersInRoom(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testUsersInRoomLeaveCallThenLeaveRoomSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + verifyNoInteractions(mockedCallParticipantListObserver); + } + + @Test + public void testUsersInRoomLeaveCallAndRoom() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + + participantListMessageListener.onUsersInRoom(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + + verify(mockedCallParticipantListObserver, only()).onCallParticipantsChanged(expectedJoined, expectedUpdated, + expectedLeft, expectedUnchanged); + } + + @Test + public void testUsersInRoomLeaveCallAndRoomSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + participantListMessageListener.onUsersInRoom(participants); + + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + expectedLeft.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + + verify(mockedCallParticipantListObserver).onCallParticipantsChanged(eq(expectedJoined), eq(expectedUpdated), + argThat(matchesExpectedLeftIgnoringOrder), eq(expectedUnchanged)); + } + + @Test + public void testUsersInRoomSeveralEventsSeveralParticipants() { + List participants = new ArrayList<>(); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 1, "theSessionId1", "theUserId1")); + participants.add(builder.newGuest(IN_CALL, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + participants.add(builder.newUser(IN_CALL, 5, "theSessionId5", "theUserId5")); + // theSessionId6 has not joined yet. + participants.add(builder.newGuest(IN_CALL | WITH_VIDEO, 7, "theSessionId7")); + participants.add(builder.newUser(DISCONNECTED, 8, "theSessionId8", "theUserId8")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 9, "theSessionId9", "theUserId9")); + + participantListMessageListener.onUsersInRoom(participants); + + callParticipantList.addObserver(mockedCallParticipantListObserver); + + participants = new ArrayList<>(); + // theSessionId1 is gone. + participants.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + participants.add(builder.newUser(DISCONNECTED, 3, "theSessionId3", "theUserId3")); + participants.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 5, "theSessionId5", "theUserId5")); + participants.add(builder.newGuest(IN_CALL | WITH_AUDIO, 6, "theSessionId6")); + participants.add(builder.newGuest(IN_CALL, 7, "theSessionId7")); + participants.add(builder.newUser(IN_CALL, 8, "theSessionId8", "theUserId8")); + participants.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId9", "theUserId9")); + + participantListMessageListener.onUsersInRoom(participants); + + expectedJoined.add(builder.newGuest(IN_CALL | WITH_AUDIO, 6, "theSessionId6")); + expectedJoined.add(builder.newUser(IN_CALL, 8, "theSessionId8", "theUserId8")); + expectedUpdated.add(builder.newUser(IN_CALL | WITH_AUDIO | WITH_VIDEO, 5, "theSessionId5", "theUserId5")); + expectedUpdated.add(builder.newGuest(IN_CALL, 7, "theSessionId7")); + expectedLeft.add(builder.newUser(DISCONNECTED, 1, "theSessionId1", "theUserId1")); + expectedLeft.add(builder.newGuest(DISCONNECTED, 2, "theSessionId2")); + expectedUnchanged.add(builder.newUser(IN_CALL, 4, "theSessionId4", "theUserId4")); + // Last ping is not seen as changed, even if it did. + expectedUnchanged.add(builder.newUser(IN_CALL | WITH_AUDIO, 42, "theSessionId9", "theUserId9")); + + verify(mockedCallParticipantListObserver).onCallParticipantsChanged(eq(expectedJoined), eq(expectedUpdated), + argThat(matchesExpectedLeftIgnoringOrder), eq(expectedUnchanged)); + } +} diff --git a/app/src/test/java/com/nextcloud/talk/call/CallParticipantListTest.java b/app/src/test/java/com/nextcloud/talk/call/CallParticipantListTest.java new file mode 100644 index 000000000..7fed0f6d8 --- /dev/null +++ b/app/src/test/java/com/nextcloud/talk/call/CallParticipantListTest.java @@ -0,0 +1,61 @@ +/* + * Nextcloud Talk application + * + * @author Daniel Calviño Sánchez + * Copyright (C) 2022 Daniel Calviño Sánchez + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package com.nextcloud.talk.call; + +import com.nextcloud.talk.signaling.SignalingMessageReceiver; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +public class CallParticipantListTest { + + private SignalingMessageReceiver mockedSignalingMessageReceiver; + + private CallParticipantList callParticipantList; + private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener; + + @Before + public void setUp() { + mockedSignalingMessageReceiver = mock(SignalingMessageReceiver.class); + + callParticipantList = new CallParticipantList(mockedSignalingMessageReceiver); + + // Get internal ParticipantListMessageListener from callParticipantList set in the + // mockedSignalingMessageReceiver. + ArgumentCaptor participantListMessageListenerArgumentCaptor = + ArgumentCaptor.forClass(SignalingMessageReceiver.ParticipantListMessageListener.class); + + verify(mockedSignalingMessageReceiver).addListener(participantListMessageListenerArgumentCaptor.capture()); + + participantListMessageListener = participantListMessageListenerArgumentCaptor.getValue(); + } + + @Test + public void testDestroy() { + callParticipantList.destroy(); + + verify(mockedSignalingMessageReceiver).removeListener(participantListMessageListener); + verifyNoMoreInteractions(mockedSignalingMessageReceiver); + } +} From ed5e8fc82eabdfc78a1997e0c78bb6e920929659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 29 Nov 2022 14:54:41 +0100 Subject: [PATCH 16/19] Use helper class to keep track of the participants in a call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As CallParticipantList starts listening on the signaling messages as soon as it is created it needs to be created and destroyed right before entering and exiting a call. Otherwise it could receive messages on other states (for example, while the "connection timeout" message is shown) and thus once the local participant joined the event would not include the other participants already in the call as joined (although they would be anyway reported as unchanged). Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index df89f8b0b..967717fe6 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -61,6 +61,7 @@ import com.nextcloud.talk.adapters.ParticipantsAdapter; import com.nextcloud.talk.api.NcApi; import com.nextcloud.talk.application.NextcloudTalkApplication; import com.nextcloud.talk.call.CallParticipant; +import com.nextcloud.talk.call.CallParticipantList; import com.nextcloud.talk.call.CallParticipantModel; import com.nextcloud.talk.data.user.model.User; import com.nextcloud.talk.databinding.CallActivityBinding; @@ -125,8 +126,8 @@ import org.webrtc.VideoTrack; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -272,27 +273,22 @@ public class CallActivity extends CallBaseActivity { private Handler screenParticipantDisplayItemManagersHandler = new Handler(Looper.getMainLooper()); - private SignalingMessageReceiver.ParticipantListMessageListener participantListMessageListener = new SignalingMessageReceiver.ParticipantListMessageListener() { - + private CallParticipantList.Observer callParticipantListObserver = new CallParticipantList.Observer() { @Override - public void onUsersInRoom(List participants) { - processUsersInRoom(participants); + public void onCallParticipantsChanged(Collection joined, Collection updated, + Collection left, Collection unchanged) { + handleCallParticipantsChanged(joined, updated, left, unchanged); } @Override - public void onParticipantsUpdate(List participants) { - processUsersInRoom(participants); - } - - @Override - public void onAllParticipantsUpdate(long inCall) { - if (inCall == Participant.InCallFlags.DISCONNECTED) { - Log.d(TAG, "A moderator ended the call for all."); - hangup(true); - } + public void onCallEndedForAll() { + Log.d(TAG, "A moderator ended the call for all."); + hangup(true); } }; + private CallParticipantList callParticipantList; + private SignalingMessageReceiver.OfferMessageListener offerMessageListener = new SignalingMessageReceiver.OfferMessageListener() { @Override public void onOffer(String sessionId, String roomType, String sdp, String nick) { @@ -1245,7 +1241,6 @@ public class CallActivity extends CallBaseActivity { @Override public void onDestroy() { - signalingMessageReceiver.removeListener(participantListMessageListener); signalingMessageReceiver.removeListener(offerMessageListener); if (localStream != null) { @@ -1379,7 +1374,6 @@ public class CallActivity extends CallBaseActivity { setupAndInitiateWebSocketsConnection(); } else { signalingMessageReceiver = internalSignalingMessageReceiver; - signalingMessageReceiver.addListener(participantListMessageListener); signalingMessageReceiver.addListener(offerMessageListener); signalingMessageSender = internalSignalingMessageSender; joinRoomAndCall(); @@ -1469,6 +1463,9 @@ public class CallActivity extends CallBaseActivity { inCallFlag += Participant.InCallFlags.WITH_VIDEO; } + callParticipantList = new CallParticipantList(signalingMessageReceiver); + callParticipantList.addObserver(callParticipantListObserver); + int apiVersion = ApiUtils.getCallApiVersion(conversationUser, new int[]{ApiUtils.APIv4, 1}); ncApi.joinCall( @@ -1583,7 +1580,6 @@ public class CallActivity extends CallBaseActivity { // Although setupAndInitiateWebSocketsConnection could be called several times the web socket is // initialized just once, so the message receiver is also initialized just once. signalingMessageReceiver = webSocketClient.getSignalingMessageReceiver(); - signalingMessageReceiver.addListener(participantListMessageListener); signalingMessageReceiver.addListener(offerMessageListener); signalingMessageSender = webSocketClient.getSignalingMessageSender(); } else { @@ -1741,6 +1737,9 @@ public class CallActivity extends CallBaseActivity { Log.d(TAG, "hangupNetworkCalls. shutDownView=" + shutDownView); int apiVersion = ApiUtils.getCallApiVersion(conversationUser, new int[]{ApiUtils.APIv4, 1}); + callParticipantList.removeObserver(callParticipantListObserver); + callParticipantList.destroy(); + ncApi.leaveCall(credentials, ApiUtils.getUrlForCall(apiVersion, baseUrl, roomToken)) .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) @@ -1778,11 +1777,9 @@ public class CallActivity extends CallBaseActivity { } } - private void processUsersInRoom(List participants) { - Log.d(TAG, "processUsersInRoom"); - List newSessions = new ArrayList<>(); - Set oldSessions = new HashSet<>(); - Map userIdsBySessionId = new HashMap<>(); + private void handleCallParticipantsChanged(Collection joined, Collection updated, + Collection left, Collection unchanged) { + Log.d(TAG, "handleCallParticipantsChanged"); hasMCU = hasExternalSignalingServer && webSocketClient != null && webSocketClient.hasMCU(); Log.d(TAG, " hasMCU is " + hasMCU); @@ -1795,58 +1792,45 @@ public class CallActivity extends CallBaseActivity { Log.d(TAG, " currentSessionId is " + currentSessionId); + List participantsInCall = new ArrayList<>(); + participantsInCall.addAll(joined); + participantsInCall.addAll(updated); + participantsInCall.addAll(unchanged); + boolean isSelfInCall = false; - for (Participant participant : participants) { + for (Participant participant : participantsInCall) { long inCallFlag = participant.getInCall(); if (!participant.getSessionId().equals(currentSessionId)) { Log.d(TAG, " inCallFlag of participant " + participant.getSessionId().substring(0, 4) + " : " + inCallFlag); - - boolean isInCall = inCallFlag != 0; - if (isInCall) { - newSessions.add(participant.getSessionId()); - } - - userIdsBySessionId.put(participant.getSessionId(), participant.getUserId()); } else { Log.d(TAG, " inCallFlag of currentSessionId: " + inCallFlag); isSelfInCall = inCallFlag != 0; - if (inCallFlag == 0 && currentCallStatus != CallStatus.LEAVING && ApplicationWideCurrentRoomHolder.getInstance().isInCall()) { - Log.d(TAG, "Most probably a moderator ended the call for all."); - hangup(true); - - return; - } } } - for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) { - if (!peerConnectionWrapper.isMCUPublisher()) { - oldSessions.add(peerConnectionWrapper.getSessionId()); - } + if (!isSelfInCall && currentCallStatus != CallStatus.LEAVING && ApplicationWideCurrentRoomHolder.getInstance().isInCall()) { + Log.d(TAG, "Most probably a moderator ended the call for all."); + hangup(true); + + return; } if (!isSelfInCall) { Log.d(TAG, "Self not in call, disconnecting from all other sessions"); - for (String sessionId : oldSessions) { - Log.d(TAG, " oldSession that will be removed is: " + sessionId); + for (Participant participant : participantsInCall) { + String sessionId = participant.getSessionId(); + Log.d(TAG, " session that will be removed is: " + sessionId); endPeerConnection(sessionId, false); } return; } - // Calculate sessions that left the call - List disconnectedSessions = new ArrayList<>(oldSessions); - disconnectedSessions.removeAll(newSessions); - - // Calculate sessions that join the call - newSessions.removeAll(oldSessions); - if (currentCallStatus == CallStatus.LEAVING) { return; } @@ -1856,11 +1840,25 @@ public class CallActivity extends CallBaseActivity { getOrCreatePeerConnectionWrapperForSessionIdAndType(webSocketClient.getSessionId(), VIDEO_STREAM_TYPE_VIDEO, true); } - for (String sessionId : newSessions) { + boolean selfJoined = false; + + for (Participant participant : joined) { + String sessionId = participant.getSessionId(); + + if (sessionId == null) { + Log.w(TAG, "Null sessionId for call participant, this should not happen: " + participant); + continue; + } + + if (sessionId.equals(currentSessionId)) { + selfJoined = true; + continue; + } + Log.d(TAG, " newSession joined: " + sessionId); getOrCreatePeerConnectionWrapperForSessionIdAndType(sessionId, VIDEO_STREAM_TYPE_VIDEO, false); - String userId = userIdsBySessionId.get(sessionId); + String userId = participant.getUserId(); if (userId != null) { callParticipants.get(sessionId).setUserId(userId); } @@ -1874,11 +1872,13 @@ public class CallActivity extends CallBaseActivity { callParticipants.get(sessionId).setNick(nick); } - if (newSessions.size() > 0 && currentCallStatus != CallStatus.IN_CONVERSATION) { + boolean othersInCall = selfJoined ? joined.size() > 1 : joined.size() > 0; + if (othersInCall && currentCallStatus != CallStatus.IN_CONVERSATION) { setCallState(CallStatus.IN_CONVERSATION); } - for (String sessionId : disconnectedSessions) { + for (Participant participant : left) { + String sessionId = participant.getSessionId(); Log.d(TAG, " oldSession that will be removed is: " + sessionId); endPeerConnection(sessionId, false); } From 2cb7572dbc7191c3a3205b98fc8d06ccd9ee2294 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 29 Nov 2022 17:14:41 +0100 Subject: [PATCH 17/19] Extract methods to add and remove call participants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 100 ++++++++++-------- 1 file changed, 56 insertions(+), 44 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 967717fe6..43705eccc 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1973,31 +1973,7 @@ public class CallActivity extends CallBaseActivity { if (!publisher) { CallParticipant callParticipant = callParticipants.get(sessionId); if (callParticipant == null) { - callParticipant = new CallParticipant(sessionId); - callParticipants.put(sessionId, callParticipant); - - SignalingMessageReceiver.CallParticipantMessageListener callParticipantMessageListener = - new CallActivityCallParticipantMessageListener(sessionId); - callParticipantMessageListeners.put(sessionId, callParticipantMessageListener); - signalingMessageReceiver.addListener(callParticipantMessageListener, sessionId); - - if (!hasExternalSignalingServer) { - OfferAnswerNickProvider offerAnswerNickProvider = new OfferAnswerNickProvider(sessionId); - offerAnswerNickProviders.put(sessionId, offerAnswerNickProvider); - signalingMessageReceiver.addListener(offerAnswerNickProvider.getVideoWebRtcMessageListener(), sessionId, "video"); - signalingMessageReceiver.addListener(offerAnswerNickProvider.getScreenWebRtcMessageListener(), sessionId, "screen"); - } - - final CallParticipantModel callParticipantModel = callParticipant.getCallParticipantModel(); - - ScreenParticipantDisplayItemManager screenParticipantDisplayItemManager = - new ScreenParticipantDisplayItemManager(callParticipantModel); - screenParticipantDisplayItemManagers.put(sessionId, screenParticipantDisplayItemManager); - callParticipantModel.addObserver(screenParticipantDisplayItemManager, screenParticipantDisplayItemManagersHandler); - - runOnUiThread(() -> { - addParticipantDisplayItem(callParticipantModel, "video"); - }); + callParticipant = addCallParticipant(sessionId); } if ("screen".equals(type)) { @@ -2017,6 +1993,36 @@ public class CallActivity extends CallBaseActivity { } } + private CallParticipant addCallParticipant(String sessionId) { + CallParticipant callParticipant = new CallParticipant(sessionId); + callParticipants.put(sessionId, callParticipant); + + SignalingMessageReceiver.CallParticipantMessageListener callParticipantMessageListener = + new CallActivityCallParticipantMessageListener(sessionId); + callParticipantMessageListeners.put(sessionId, callParticipantMessageListener); + signalingMessageReceiver.addListener(callParticipantMessageListener, sessionId); + + if (!hasExternalSignalingServer) { + OfferAnswerNickProvider offerAnswerNickProvider = new OfferAnswerNickProvider(sessionId); + offerAnswerNickProviders.put(sessionId, offerAnswerNickProvider); + signalingMessageReceiver.addListener(offerAnswerNickProvider.getVideoWebRtcMessageListener(), sessionId, "video"); + signalingMessageReceiver.addListener(offerAnswerNickProvider.getScreenWebRtcMessageListener(), sessionId, "screen"); + } + + final CallParticipantModel callParticipantModel = callParticipant.getCallParticipantModel(); + + ScreenParticipantDisplayItemManager screenParticipantDisplayItemManager = + new ScreenParticipantDisplayItemManager(callParticipantModel); + screenParticipantDisplayItemManagers.put(sessionId, screenParticipantDisplayItemManager); + callParticipantModel.addObserver(screenParticipantDisplayItemManager, screenParticipantDisplayItemManagersHandler); + + runOnUiThread(() -> { + addParticipantDisplayItem(callParticipantModel, "video"); + }); + + return callParticipant; + } + private List getPeerConnectionWrapperListForSessionId(String sessionId) { List internalList = new ArrayList<>(); for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) { @@ -2055,28 +2061,34 @@ public class CallActivity extends CallBaseActivity { } if (!justScreen) { - CallParticipant callParticipant = callParticipants.remove(sessionId); - if (callParticipant != null) { - ScreenParticipantDisplayItemManager screenParticipantDisplayItemManager = - screenParticipantDisplayItemManagers.remove(sessionId); - callParticipant.getCallParticipantModel().removeObserver(screenParticipantDisplayItemManager); - - callParticipant.destroy(); - - SignalingMessageReceiver.CallParticipantMessageListener listener = callParticipantMessageListeners.remove(sessionId); - signalingMessageReceiver.removeListener(listener); - - OfferAnswerNickProvider offerAnswerNickProvider = offerAnswerNickProviders.remove(sessionId); - if (offerAnswerNickProvider != null) { - signalingMessageReceiver.removeListener(offerAnswerNickProvider.getVideoWebRtcMessageListener()); - signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener()); - } - - runOnUiThread(() -> removeParticipantDisplayItem(sessionId, "video")); - } + removeCallParticipant(sessionId); } } + private void removeCallParticipant(String sessionId) { + CallParticipant callParticipant = callParticipants.remove(sessionId); + if (callParticipant == null) { + return; + } + + ScreenParticipantDisplayItemManager screenParticipantDisplayItemManager = + screenParticipantDisplayItemManagers.remove(sessionId); + callParticipant.getCallParticipantModel().removeObserver(screenParticipantDisplayItemManager); + + callParticipant.destroy(); + + SignalingMessageReceiver.CallParticipantMessageListener listener = callParticipantMessageListeners.remove(sessionId); + signalingMessageReceiver.removeListener(listener); + + OfferAnswerNickProvider offerAnswerNickProvider = offerAnswerNickProviders.remove(sessionId); + if (offerAnswerNickProvider != null) { + signalingMessageReceiver.removeListener(offerAnswerNickProvider.getVideoWebRtcMessageListener()); + signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener()); + } + + runOnUiThread(() -> removeParticipantDisplayItem(sessionId, "video")); + } + private void removeParticipantDisplayItem(String sessionId, String videoStreamType) { Log.d(TAG, "removeParticipantDisplayItem"); ParticipantDisplayItem participantDisplayItem = participantDisplayItems.remove(sessionId + "-" + videoStreamType); From 9ae969b0f82c30909c0189fb5d3432d897b9908b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 29 Nov 2022 20:34:32 +0100 Subject: [PATCH 18/19] Split call participants and peer connections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of trying to create a video peer connection for any joined participant now only a call participant is created for any joined participant, and a video peer connection is created only for those participants that are publishing audio or video. If a call participants does not have a video peer connection the call participant is now seen as "connected" from the UI, as there is no need to show a progress bar for that participant. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 46 +++++++++++++++---- .../talk/adapters/ParticipantDisplayItem.java | 5 +- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 43705eccc..ee07afe0b 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1721,14 +1721,22 @@ public class CallActivity extends CallBaseActivity { } } - List sessionIdsToEnd = new ArrayList(peerConnectionWrapperList.size()); + List peerConnectionIdsToEnd = new ArrayList(peerConnectionWrapperList.size()); for (PeerConnectionWrapper wrapper : peerConnectionWrapperList) { - sessionIdsToEnd.add(wrapper.getSessionId()); + peerConnectionIdsToEnd.add(wrapper.getSessionId()); } - for (String sessionId : sessionIdsToEnd) { + for (String sessionId : peerConnectionIdsToEnd) { endPeerConnection(sessionId, false); } + List callParticipantIdsToEnd = new ArrayList(peerConnectionWrapperList.size()); + for (CallParticipant callParticipant : callParticipants.values()) { + callParticipantIdsToEnd.add(callParticipant.getCallParticipantModel().getSessionId()); + } + for (String sessionId : callParticipantIdsToEnd) { + removeCallParticipant(sessionId); + } + hangupNetworkCalls(shutDownView); ApplicationWideCurrentRoomHolder.getInstance().setInCall(false); } @@ -1798,6 +1806,7 @@ public class CallActivity extends CallBaseActivity { participantsInCall.addAll(unchanged); boolean isSelfInCall = false; + Participant selfParticipant = null; for (Participant participant : participantsInCall) { long inCallFlag = participant.getInCall(); @@ -1809,6 +1818,7 @@ public class CallActivity extends CallBaseActivity { } else { Log.d(TAG, " inCallFlag of currentSessionId: " + inCallFlag); isSelfInCall = inCallFlag != 0; + selfParticipant = participant; } } @@ -1826,6 +1836,7 @@ public class CallActivity extends CallBaseActivity { String sessionId = participant.getSessionId(); Log.d(TAG, " session that will be removed is: " + sessionId); endPeerConnection(sessionId, false); + removeCallParticipant(sessionId); } return; @@ -1841,6 +1852,7 @@ public class CallActivity extends CallBaseActivity { } boolean selfJoined = false; + boolean selfParticipantHasAudioOrVideo = participantInCallFlagsHaveAudioOrVideo(selfParticipant); for (Participant participant : joined) { String sessionId = participant.getSessionId(); @@ -1856,7 +1868,8 @@ public class CallActivity extends CallBaseActivity { } Log.d(TAG, " newSession joined: " + sessionId); - getOrCreatePeerConnectionWrapperForSessionIdAndType(sessionId, VIDEO_STREAM_TYPE_VIDEO, false); + + CallParticipant callParticipant = addCallParticipant(sessionId); String userId = participant.getUserId(); if (userId != null) { @@ -1870,6 +1883,17 @@ public class CallActivity extends CallBaseActivity { nick = offerAnswerNickProviders.get(sessionId) != null ? offerAnswerNickProviders.get(sessionId).getNick() : ""; } callParticipants.get(sessionId).setNick(nick); + + boolean participantHasAudioOrVideo = participantInCallFlagsHaveAudioOrVideo(participant); + + // FIXME Without MCU, PeerConnectionWrapper only sends an offer if the local session ID is higher than the + // remote session ID. However, if the other participant does not have audio nor video that participant + // will not send an offer, so no connection is actually established when the remote participant has a + // higher session ID but is not publishing media. + if ((hasMCU && participantHasAudioOrVideo) || + (!hasMCU && selfParticipantHasAudioOrVideo && (!participantHasAudioOrVideo || sessionId.compareTo(currentSessionId) < 0))) { + getOrCreatePeerConnectionWrapperForSessionIdAndType(sessionId, VIDEO_STREAM_TYPE_VIDEO, false); + } } boolean othersInCall = selfJoined ? joined.size() > 1 : joined.size() > 0; @@ -1881,9 +1905,19 @@ public class CallActivity extends CallBaseActivity { String sessionId = participant.getSessionId(); Log.d(TAG, " oldSession that will be removed is: " + sessionId); endPeerConnection(sessionId, false); + removeCallParticipant(sessionId); } } + private boolean participantInCallFlagsHaveAudioOrVideo(Participant participant) { + if (participant == null) { + return false; + } + + return (participant.getInCall() & Participant.InCallFlags.WITH_AUDIO) > 0 || + (!isVoiceOnlyCall && (participant.getInCall() & Participant.InCallFlags.WITH_VIDEO) > 0); + } + private void deletePeerConnection(PeerConnectionWrapper peerConnectionWrapper) { peerConnectionWrapper.removePeerConnection(); peerConnectionWrapperList.remove(peerConnectionWrapper); @@ -2059,10 +2093,6 @@ public class CallActivity extends CallBaseActivity { } } } - - if (!justScreen) { - removeCallParticipant(sessionId); - } } private void removeCallParticipant(String sessionId) { diff --git a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java index 6a9912ead..418afa1a5 100644 --- a/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java +++ b/app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java @@ -95,7 +95,10 @@ public class ParticipantDisplayItem { public boolean isConnected() { return iceConnectionState == PeerConnection.IceConnectionState.CONNECTED || - iceConnectionState == PeerConnection.IceConnectionState.COMPLETED; + iceConnectionState == PeerConnection.IceConnectionState.COMPLETED || + // If there is no connection state that means that no connection is needed, so it is a special case that is + // also seen as "connected". + iceConnectionState == null; } public String getNick() { From 67e259f79257fe5cd7679f872f7546737d2d5fbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 29 Nov 2022 20:49:29 +0100 Subject: [PATCH 19/19] Simplify ending the peer connections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The peer connections will be of either "video" or "screen" type, so they can be simply removed based on the session id and an explicit type. Signed-off-by: Daniel Calviño Sánchez --- .../talk/activities/CallActivity.java | 67 +++++++------------ 1 file changed, 25 insertions(+), 42 deletions(-) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index ee07afe0b..49fcb6d6a 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -1726,7 +1726,8 @@ public class CallActivity extends CallBaseActivity { peerConnectionIdsToEnd.add(wrapper.getSessionId()); } for (String sessionId : peerConnectionIdsToEnd) { - endPeerConnection(sessionId, false); + endPeerConnection(sessionId, "video"); + endPeerConnection(sessionId, "screen"); } List callParticipantIdsToEnd = new ArrayList(peerConnectionWrapperList.size()); @@ -1835,7 +1836,8 @@ public class CallActivity extends CallBaseActivity { for (Participant participant : participantsInCall) { String sessionId = participant.getSessionId(); Log.d(TAG, " session that will be removed is: " + sessionId); - endPeerConnection(sessionId, false); + endPeerConnection(sessionId, "video"); + endPeerConnection(sessionId, "screen"); removeCallParticipant(sessionId); } @@ -1904,7 +1906,8 @@ public class CallActivity extends CallBaseActivity { for (Participant participant : left) { String sessionId = participant.getSessionId(); Log.d(TAG, " oldSession that will be removed is: " + sessionId); - endPeerConnection(sessionId, false); + endPeerConnection(sessionId, "video"); + endPeerConnection(sessionId, "screen"); removeCallParticipant(sessionId); } } @@ -1918,11 +1921,6 @@ public class CallActivity extends CallBaseActivity { (!isVoiceOnlyCall && (participant.getInCall() & Participant.InCallFlags.WITH_VIDEO) > 0); } - private void deletePeerConnection(PeerConnectionWrapper peerConnectionWrapper) { - peerConnectionWrapper.removePeerConnection(); - peerConnectionWrapperList.remove(peerConnectionWrapper); - } - private PeerConnectionWrapper getPeerConnectionWrapperForSessionIdAndType(String sessionId, String type) { for (PeerConnectionWrapper wrapper : peerConnectionWrapperList) { if (wrapper.getSessionId().equals(sessionId) @@ -2057,42 +2055,27 @@ public class CallActivity extends CallBaseActivity { return callParticipant; } - private List getPeerConnectionWrapperListForSessionId(String sessionId) { - List internalList = new ArrayList<>(); - for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) { - if (peerConnectionWrapper.getSessionId().equals(sessionId)) { - internalList.add(peerConnectionWrapper); + private void endPeerConnection(String sessionId, String type) { + PeerConnectionWrapper peerConnectionWrapper = getPeerConnectionWrapperForSessionIdAndType(sessionId, type); + if (peerConnectionWrapper == null) { + return; + } + + if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { + peerConnectionWrapper.removeObserver(selfPeerConnectionObserver); + } + + CallParticipant callParticipant = callParticipants.get(sessionId); + if (callParticipant != null) { + if ("screen".equals(type)) { + callParticipant.setScreenPeerConnectionWrapper(null); + } else { + callParticipant.setPeerConnectionWrapper(null); } } - return internalList; - } - - private void endPeerConnection(String sessionId, boolean justScreen) { - List peerConnectionWrappers; - if (!(peerConnectionWrappers = getPeerConnectionWrapperListForSessionId(sessionId)).isEmpty()) { - for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrappers) { - if (peerConnectionWrapper.getSessionId().equals(sessionId)) { - if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) { - peerConnectionWrapper.removeObserver(selfPeerConnectionObserver); - } - - String videoStreamType = peerConnectionWrapper.getVideoStreamType(); - if (VIDEO_STREAM_TYPE_SCREEN.equals(videoStreamType) || !justScreen) { - CallParticipant callParticipant = callParticipants.get(sessionId); - if (callParticipant != null) { - if ("screen".equals(videoStreamType)) { - callParticipant.setScreenPeerConnectionWrapper(null); - } else { - callParticipant.setPeerConnectionWrapper(null); - } - } - - deletePeerConnection(peerConnectionWrapper); - } - } - } - } + peerConnectionWrapper.removePeerConnection(); + peerConnectionWrapperList.remove(peerConnectionWrapper); } private void removeCallParticipant(String sessionId) { @@ -2599,7 +2582,7 @@ public class CallActivity extends CallBaseActivity { @Override public void onUnshareScreen() { - endPeerConnection(sessionId, true); + endPeerConnection(sessionId, "screen"); } }