Merge pull request #2390 from nextcloud/provide-visual-feeback-when-a-participant-is-not-connected

Provide visual feeback when a participant is not connected

app/src/main/java/com/nextcloud/talk/activities/CallActivity.java

@@ -1843,6 +1843,14 @@ public class CallActivity extends CallBaseActivity {
         for (String sessionId : newSessions) {
             Log.d(TAG, "   newSession joined: " + sessionId);
             getOrCreatePeerConnectionWrapperForSessionIdAndType(sessionId, VIDEO_STREAM_TYPE_VIDEO, false);
+
+            runOnUiThread(() -> {
+                setupVideoStreamForLayout(
+                    null,
+                    sessionId,
+                    false,
+                    VIDEO_STREAM_TYPE_VIDEO);
+            });
         }
 
         if (newSessions.size() > 0 && !currentCallStatus.equals(CallStatus.IN_CONVERSATION)) {
@@ -2019,6 +2027,17 @@ public class CallActivity extends CallBaseActivity {
         updateSelfVideoViewPosition();
     }
 
+    private void updateSelfVideoViewConnected(boolean connected) {
+        // FIXME In voice only calls there is no video view, so the progress bar would appear floating in the middle of
+        // nowhere. However, a way to signal that the local participant is not connected to the HPB is still need in
+        // that case.
+        if (!connected && !isVoiceOnlyCall) {
+            binding.selfVideoViewProgressBar.setVisibility(View.VISIBLE);
+        } else {
+            binding.selfVideoViewProgressBar.setVisibility(View.GONE);
+        }
+    }
+
     private void updateSelfVideoViewPosition() {
         Log.d(TAG, "updateSelfVideoViewPosition");
         if (!isInPipMode) {
@@ -2061,6 +2080,22 @@ public class CallActivity extends CallBaseActivity {
         String sessionId = peerConnectionEvent.getSessionId();
 
         if (peerConnectionEvent.getPeerConnectionEventType() ==
+            PeerConnectionEvent.PeerConnectionEventType.PEER_CONNECTED) {
+            if (webSocketClient != null && webSocketClient.getSessionId() == sessionId) {
+                updateSelfVideoViewConnected(true);
+            } else if (participantDisplayItems.get(sessionId) != null) {
+                participantDisplayItems.get(sessionId).setConnected(true);
+                participantsAdapter.notifyDataSetChanged();
+            }
+        } else if (peerConnectionEvent.getPeerConnectionEventType() ==
+            PeerConnectionEvent.PeerConnectionEventType.PEER_DISCONNECTED) {
+            if (webSocketClient != null && webSocketClient.getSessionId() == sessionId) {
+                updateSelfVideoViewConnected(false);
+            } else if (participantDisplayItems.get(sessionId) != null) {
+                participantDisplayItems.get(sessionId).setConnected(false);
+                participantsAdapter.notifyDataSetChanged();
+            }
+        } else if (peerConnectionEvent.getPeerConnectionEventType() ==
             PeerConnectionEvent.PeerConnectionEventType.PEER_CLOSED) {
             endPeerConnection(sessionId, VIDEO_STREAM_TYPE_SCREEN.equals(peerConnectionEvent.getVideoStreamType()));
         } else if (peerConnectionEvent.getPeerConnectionEventType() ==
@@ -2081,23 +2116,20 @@ public class CallActivity extends CallBaseActivity {
             PeerConnectionEvent.PeerConnectionEventType.NICK_CHANGE) {
             if (participantDisplayItems.get(sessionId) != null) {
                 participantDisplayItems.get(sessionId).setNick(peerConnectionEvent.getNick());
-            }
                 participantsAdapter.notifyDataSetChanged();
-
+            }
         } else if (peerConnectionEvent.getPeerConnectionEventType() ==
             PeerConnectionEvent.PeerConnectionEventType.VIDEO_CHANGE && !isVoiceOnlyCall) {
             if (participantDisplayItems.get(sessionId) != null) {
                 participantDisplayItems.get(sessionId).setStreamEnabled(peerConnectionEvent.getChangeValue());
-            }
                 participantsAdapter.notifyDataSetChanged();
-
+            }
         } else if (peerConnectionEvent.getPeerConnectionEventType() ==
             PeerConnectionEvent.PeerConnectionEventType.AUDIO_CHANGE) {
             if (participantDisplayItems.get(sessionId) != null) {
                 participantDisplayItems.get(sessionId).setAudioEnabled(peerConnectionEvent.getChangeValue());
-            }
                 participantsAdapter.notifyDataSetChanged();
-
+            }
         } else if (peerConnectionEvent.getPeerConnectionEventType() ==
             PeerConnectionEvent.PeerConnectionEventType.PUBLISHER_FAILED) {
             currentCallStatus = CallStatus.PUBLISHER_FAILED;
@@ -2253,12 +2285,20 @@ public class CallActivity extends CallBaseActivity {
                                            String session,
                                            boolean videoStreamEnabled,
                                            String videoStreamType) {
+        PeerConnectionWrapper peerConnectionWrapper = getPeerConnectionWrapperForSessionIdAndType(session,
+                                                                                                  videoStreamType);
+
+        boolean connected = false;
+        if (peerConnectionWrapper != null) {
+            PeerConnection.IceConnectionState iceConnectionState = peerConnectionWrapper.getPeerConnection().iceConnectionState();
+            connected = iceConnectionState == PeerConnection.IceConnectionState.CONNECTED ||
+                        iceConnectionState == PeerConnection.IceConnectionState.COMPLETED;
+        }
+
         String nick;
         if (hasExternalSignalingServer) {
             nick = webSocketClient.getDisplayNameForSession(session);
         } else {
-            PeerConnectionWrapper peerConnectionWrapper = getPeerConnectionWrapperForSessionIdAndType(session,
-                                                                                                      videoStreamType);
             nick = peerConnectionWrapper != null ? peerConnectionWrapper.getNick() : "";
         }
 
@@ -2282,6 +2322,7 @@ public class CallActivity extends CallBaseActivity {
 
         ParticipantDisplayItem participantDisplayItem = new ParticipantDisplayItem(userId,
                                                                                    session,
+                                                                                   connected,
                                                                                    nick,
                                                                                    urlForAvatar,
                                                                                    mediaStream,

app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java

@@ -6,6 +6,7 @@ import org.webrtc.MediaStream;
 public class ParticipantDisplayItem {
     private String userId;
     private String session;
+    private boolean connected;
     private String nick;
     private String urlForAvatar;
     private MediaStream mediaStream;
@@ -14,9 +15,10 @@ public class ParticipantDisplayItem {
     private EglBase rootEglBase;
     private boolean isAudioEnabled;
 
-    public ParticipantDisplayItem(String userId, String session, String nick, String urlForAvatar, MediaStream mediaStream, String streamType, boolean streamEnabled, EglBase rootEglBase) {
+    public ParticipantDisplayItem(String userId, String session, boolean connected, String nick, String urlForAvatar, MediaStream mediaStream, String streamType, boolean streamEnabled, EglBase rootEglBase) {
         this.userId = userId;
         this.session = session;
+        this.connected = connected;
         this.nick = nick;
         this.urlForAvatar = urlForAvatar;
         this.mediaStream = mediaStream;
@@ -41,6 +43,14 @@ public class ParticipantDisplayItem {
         this.session = session;
     }
 
+    public boolean isConnected() {
+        return connected;
+    }
+
+    public void setConnected(boolean connected) {
+        this.connected = connected;
+    }
+
     public String getNick() {
         return nick;
     }

app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java

@@ -8,6 +8,7 @@ import android.view.ViewGroup;
 import android.widget.BaseAdapter;
 import android.widget.ImageView;
 import android.widget.LinearLayout;
+import android.widget.ProgressBar;
 import android.widget.RelativeLayout;
 import android.widget.TextView;
 
@@ -96,6 +97,13 @@ public class ParticipantsAdapter extends BaseAdapter {
             surfaceViewRenderer = convertView.findViewById(R.id.surface_view);
         }
 
+        ProgressBar progressBar = convertView.findViewById(R.id.participant_progress_bar);
+        if (!participantDisplayItem.isConnected()) {
+            progressBar.setVisibility(View.VISIBLE);
+        } else {
+            progressBar.setVisibility(View.GONE);
+        }
+
         ViewGroup.LayoutParams layoutParams = convertView.getLayoutParams();
         layoutParams.height = scaleGridViewItemHeight();
         convertView.setLayoutParams(layoutParams);

app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java

@@ -120,6 +120,6 @@ public class PeerConnectionEvent {
     }
 
     public enum PeerConnectionEventType {
-        PEER_CONNECTED, PEER_CLOSED, SENSOR_FAR, SENSOR_NEAR, NICK_CHANGE, AUDIO_CHANGE, VIDEO_CHANGE, PUBLISHER_FAILED
+        PEER_CONNECTED, PEER_DISCONNECTED, PEER_CLOSED, SENSOR_FAR, SENSOR_NEAR, NICK_CHANGE, AUDIO_CHANGE, VIDEO_CHANGE, PUBLISHER_FAILED
     }
 }

app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java

@@ -344,6 +344,8 @@ public class PeerConnectionWrapper {
 
             Log.d("iceConnectionChangeTo: ", iceConnectionState.name() + " over " + peerConnection.hashCode() + " " + sessionId);
             if (iceConnectionState.equals(PeerConnection.IceConnectionState.CONNECTED)) {
+                EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType.PEER_CONNECTED,
+                                                                   sessionId, null, null, null));
 
                 if (!isMCUPublisher) {
                     EventBus.getDefault().post(new MediaStreamEvent(remoteStream, sessionId, videoStreamType));
@@ -352,11 +354,20 @@ public class PeerConnectionWrapper {
                 if (hasInitiated) {
                     sendInitialMediaStatus();
                 }
-
+            } else if (iceConnectionState.equals(PeerConnection.IceConnectionState.COMPLETED)) {
+                EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType.PEER_CONNECTED,
+                                                                   sessionId, null, null, null));
             } else if (iceConnectionState.equals(PeerConnection.IceConnectionState.CLOSED)) {
                 EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
                         .PEER_CLOSED, sessionId, null, null, videoStreamType));
+            } else if (iceConnectionState.equals(PeerConnection.IceConnectionState.DISCONNECTED) ||
+                    iceConnectionState.equals(PeerConnection.IceConnectionState.NEW) ||
+                    iceConnectionState.equals(PeerConnection.IceConnectionState.CHECKING)) {
+                EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType.PEER_DISCONNECTED,
+                                                                   sessionId, null, null, null));
             } else if (iceConnectionState.equals(PeerConnection.IceConnectionState.FAILED)) {
+                EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType.PEER_DISCONNECTED,
+                                                                   sessionId, null, null, null));
                 if (isMCUPublisher) {
                     EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType.PUBLISHER_FAILED, sessionId, null, null, null));
                 }

app/src/main/res/layout/call_activity.xml

@@ -77,6 +77,14 @@
                     android:layout_marginBottom="20dp"
                     app:placeholderImage="@drawable/ic_switch_video_white_24px"
                     app:roundAsCircle="true" />
+
+                <ProgressBar
+                    android:id="@+id/selfVideoViewProgressBar"
+                    style="?android:attr/progressBarStyle"
+                    android:layout_width="@dimen/call_self_participant_progress_bar_size"
+                    android:layout_height="@dimen/call_self_participant_progress_bar_size"
+                    android:layout_gravity="center"
+                    android:visibility="gone" />
             </FrameLayout>
 
             <LinearLayout

app/src/main/res/layout/call_item.xml

@@ -71,4 +71,11 @@
         android:visibility="invisible"
         tools:visibility="visible" />
 
+    <ProgressBar
+        android:id="@+id/participant_progress_bar"
+        style="?android:attr/progressBarStyle"
+        android:layout_width="@dimen/call_participant_progress_bar_size"
+        android:layout_height="@dimen/call_participant_progress_bar_size"
+        android:layout_centerInParent="@bool/value_true" />
+
 </RelativeLayout>

app/src/main/res/values/dimens.xml

@@ -67,6 +67,8 @@
     <dimen name="call_self_video_short_side_length">80dp</dimen>
     <dimen name="call_grid_item_min_height">180dp</dimen>
     <dimen name="call_controls_height">110dp</dimen>
+    <dimen name="call_participant_progress_bar_size">48dp</dimen>
+    <dimen name="call_self_participant_progress_bar_size">48dp</dimen>
     <dimen name="zero">0dp</dimen>
 
     <dimen name="online_status_item_height">52dp</dimen>