Merge pull request #2721 from nextcloud/show-raised-hands-by-remote-participants

Show raised hands by remote participants

app/src/main/java/com/nextcloud/talk/activities/CallActivity.java

@@ -2124,7 +2124,7 @@ public class CallActivity extends CallBaseActivity {
     }
 
     private CallParticipant addCallParticipant(String sessionId) {
-        CallParticipant callParticipant = new CallParticipant(sessionId);
+        CallParticipant callParticipant = new CallParticipant(sessionId, signalingMessageReceiver);
         callParticipants.put(sessionId, callParticipant);
 
         SignalingMessageReceiver.CallParticipantMessageListener callParticipantMessageListener =
@@ -2679,6 +2679,20 @@ public class CallActivity extends CallBaseActivity {
             this.sessionId = sessionId;
         }
 
+        @Override
+        public void onRaiseHand(boolean state, long timestamp) {
+            if (state) {
+                CallParticipant participant = callParticipants.get(sessionId);
+                if (participant != null) {
+                    String nick = participant.getCallParticipantModel().getNick();
+                    runOnUiThread(() -> Toast.makeText(
+                        context,
+                        String.format(context.getResources().getString(R.string.nc_call_raised_hand), nick),
+                        Toast.LENGTH_LONG).show());
+                }
+            }
+        }
+
         @Override
         public void onUnshareScreen() {
             endPeerConnection(sessionId, "screen");

app/src/main/java/com/nextcloud/talk/adapters/ParticipantDisplayItem.java

@@ -5,6 +5,7 @@ import android.os.Looper;
 import android.text.TextUtils;
 
 import com.nextcloud.talk.call.CallParticipantModel;
+import com.nextcloud.talk.call.RaisedHand;
 import com.nextcloud.talk.utils.ApiUtils;
 
 import org.webrtc.EglBase;
@@ -42,6 +43,7 @@ public class ParticipantDisplayItem {
     private MediaStream mediaStream;
     private boolean streamEnabled;
     private boolean isAudioEnabled;
+    private RaisedHand raisedHand;
 
     public ParticipantDisplayItem(String baseUrl, String defaultGuestNick, EglBase rootEglBase, String streamType,
                                   CallParticipantModel callParticipantModel) {
@@ -82,6 +84,8 @@ public class ParticipantDisplayItem {
                 callParticipantModel.isVideoAvailable() : false;
         }
 
+        raisedHand = callParticipantModel.getRaisedHand();
+
         participantDisplayItemNotifier.notifyChange();
     }
 
@@ -129,6 +133,10 @@ public class ParticipantDisplayItem {
         return isAudioEnabled;
     }
 
+    public RaisedHand getRaisedHand() {
+        return raisedHand;
+    }
+
     public void addObserver(Observer observer) {
         participantDisplayItemNotifier.addObserver(observer);
     }
@@ -148,6 +156,7 @@ public class ParticipantDisplayItem {
                 ", streamType='" + streamType + '\'' +
                 ", streamEnabled=" + streamEnabled +
                 ", rootEglBase=" + rootEglBase +
+                ", raisedHand=" + raisedHand +
                 '}';
     }
 }

app/src/main/java/com/nextcloud/talk/adapters/ParticipantsAdapter.java

@@ -143,11 +143,17 @@ public class ParticipantsAdapter extends BaseAdapter {
         if (!participantDisplayItem.isAudioEnabled()) {
             audioOffView.setVisibility(View.VISIBLE);
         } else {
-            audioOffView.setVisibility(View.INVISIBLE);
+            audioOffView.setVisibility(View.GONE);
+        }
+
+        ImageView raisedHandView = convertView.findViewById(R.id.raised_hand);
+        if (participantDisplayItem.getRaisedHand() != null && participantDisplayItem.getRaisedHand().getState()) {
+            raisedHandView.setVisibility(View.VISIBLE);
+        } else {
+            raisedHandView.setVisibility(View.GONE);
         }
 
         return convertView;
-
     }
 
     private boolean hasVideoStream(ParticipantDisplayItem participantDisplayItem, MediaStream mediaStream) {

app/src/main/java/com/nextcloud/talk/call/CallParticipant.java

@@ -19,6 +19,7 @@
  */
 package com.nextcloud.talk.call;
 
+import com.nextcloud.talk.signaling.SignalingMessageReceiver;
 import com.nextcloud.talk.webrtc.PeerConnectionWrapper;
 
 import org.webrtc.MediaStream;
@@ -32,6 +33,18 @@ import org.webrtc.PeerConnection;
  */
 public class CallParticipant {
 
+    private final SignalingMessageReceiver.CallParticipantMessageListener callParticipantMessageListener =
+            new SignalingMessageReceiver.CallParticipantMessageListener() {
+        @Override
+        public void onRaiseHand(boolean state, long timestamp) {
+            callParticipantModel.setRaisedHand(state, timestamp);
+        }
+
+        @Override
+        public void onUnshareScreen() {
+        }
+    };
+
     private final PeerConnectionWrapper.PeerConnectionObserver peerConnectionObserver =
             new PeerConnectionWrapper.PeerConnectionObserver() {
         @Override
@@ -99,14 +112,21 @@ public class CallParticipant {
 
     private final MutableCallParticipantModel callParticipantModel;
 
+    private final SignalingMessageReceiver signalingMessageReceiver;
+
     private PeerConnectionWrapper peerConnectionWrapper;
     private PeerConnectionWrapper screenPeerConnectionWrapper;
 
-    public CallParticipant(String sessionId) {
+    public CallParticipant(String sessionId, SignalingMessageReceiver signalingMessageReceiver) {
         callParticipantModel = new MutableCallParticipantModel(sessionId);
+
+        this.signalingMessageReceiver = signalingMessageReceiver;
+        signalingMessageReceiver.addListener(callParticipantMessageListener, sessionId);
     }
 
     public void destroy() {
+        signalingMessageReceiver.removeListener(callParticipantMessageListener);
+
         if (peerConnectionWrapper != null) {
             peerConnectionWrapper.removeObserver(peerConnectionObserver);
             peerConnectionWrapper.removeListener(dataChannelMessageListener);

app/src/main/java/com/nextcloud/talk/call/CallParticipantModel.java

@@ -29,6 +29,9 @@ import java.util.Objects;
 /**
  * Read-only data model for (remote) call participants.
  *
+ * If the hand was never raised null is returned by "getRaisedHand()". Otherwise a RaisedHand object is returned with
+ * the current state (raised or not) and the timestamp when the raised hand state last changed.
+ *
  * The received audio and video are available only if the participant is sending them and also has them enabled.
  * Before a connection is established it is not known whether audio and video are available or not, so null is returned
  * in that case (therefore it should not be autoboxed to a plain boolean without checking that).
@@ -72,6 +75,8 @@ public class CallParticipantModel {
     protected Data<String> userId;
     protected Data<String> nick;
 
+    protected Data<RaisedHand> raisedHand;
+
     protected Data<PeerConnection.IceConnectionState> iceConnectionState;
     protected Data<MediaStream> mediaStream;
     protected Data<Boolean> audioAvailable;
@@ -86,6 +91,8 @@ public class CallParticipantModel {
         this.userId = new Data<>();
         this.nick = new Data<>();
 
+        this.raisedHand = new Data<>();
+
         this.iceConnectionState = new Data<>();
         this.mediaStream = new Data<>();
         this.audioAvailable = new Data<>();
@@ -107,6 +114,10 @@ public class CallParticipantModel {
         return nick.getValue();
     }
 
+    public RaisedHand getRaisedHand() {
+        return raisedHand.getValue();
+    }
+
     public PeerConnection.IceConnectionState getIceConnectionState() {
         return iceConnectionState.getValue();
     }

app/src/main/java/com/nextcloud/talk/call/MutableCallParticipantModel.java

@@ -41,6 +41,10 @@ public class MutableCallParticipantModel extends CallParticipantModel {
         this.nick.setValue(nick);
     }
 
+    public void setRaisedHand(boolean state, long timestamp) {
+        this.raisedHand.setValue(new RaisedHand(state, timestamp));
+    }
+
     public void setIceConnectionState(PeerConnection.IceConnectionState iceConnectionState) {
         this.iceConnectionState.setValue(iceConnectionState);
     }

app/src/main/java/com/nextcloud/talk/call/RaisedHand.kt

@@ -0,0 +1,22 @@
+/*
+ * Nextcloud Talk application
+ *
+ * @author Daniel Calviño Sánchez
+ * Copyright (C) 2023 Daniel Calviño Sánchez <danxuliu@gmail.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package com.nextcloud.talk.call
+
+data class RaisedHand(val state: Boolean, val timestamp: Long)

app/src/main/java/com/nextcloud/talk/models/json/signaling/NCMessagePayload.kt

@@ -38,8 +38,12 @@ data class NCMessagePayload(
     @JsonField(name = ["candidate"])
     var iceCandidate: NCIceCandidate? = null,
     @JsonField(name = ["name"])
-    var name: String? = null
+    var name: String? = null,
+    @JsonField(name = ["state"])
+    var state: Boolean? = null,
+    @JsonField(name = ["timestamp"])
+    var timestamp: Long? = null
 ) : Parcelable {
     // This constructor is added to work with the 'com.bluelinelabs.logansquare.annotation.JsonObject'
-    constructor() : this(null, null, null, null, null)
+    constructor() : this(null, null, null, null, null, null, null)
 }

app/src/main/java/com/nextcloud/talk/signaling/CallParticipantMessageNotifier.java

@@ -87,6 +87,12 @@ class CallParticipantMessageNotifier {
         return callParticipantMessageListeners;
     }
 
+    public synchronized void notifyRaiseHand(String sessionId, boolean state, long timestamp) {
+        for (SignalingMessageReceiver.CallParticipantMessageListener listener : getListenersFor(sessionId)) {
+            listener.onRaiseHand(state, timestamp);
+        }
+    }
+
     public synchronized void notifyUnshareScreen(String sessionId) {
         for (SignalingMessageReceiver.CallParticipantMessageListener listener : getListenersFor(sessionId)) {
             listener.onUnshareScreen();

app/src/main/java/com/nextcloud/talk/signaling/SignalingMessageReceiver.java

@@ -128,6 +128,7 @@ public abstract class SignalingMessageReceiver {
      * message on the call participant.
      */
     public interface CallParticipantMessageListener {
+        void onRaiseHand(boolean state, long timestamp);
         void onUnshareScreen();
     }
 
@@ -415,6 +416,63 @@ public abstract class SignalingMessageReceiver {
         String sessionId = signalingMessage.getFrom();
         String roomType = signalingMessage.getRoomType();
 
+        if ("raiseHand".equals(type)) {
+            // Message schema (external signaling server):
+            // {
+            //     "type": "message",
+            //     "message": {
+            //         "sender": {
+            //             ...
+            //         },
+            //         "data": {
+            //             "to": #STRING#,
+            //             "sid": #STRING#,
+            //             "roomType": "video",
+            //             "type": "raiseHand",
+            //             "payload": {
+            //                 "state": #BOOLEAN#,
+            //                 "timestamp": #LONG#,
+            //             },
+            //             "from": #STRING#,
+            //         },
+            //     },
+            // }
+            //
+            // Message schema (internal signaling server):
+            // {
+            //     "type": "message",
+            //     "data": {
+            //         "to": #STRING#,
+            //         "sid": #STRING#,
+            //         "roomType": "video",
+            //         "type": "raiseHand",
+            //         "payload": {
+            //             "state": #BOOLEAN#,
+            //             "timestamp": #LONG#,
+            //         },
+            //         "from": #STRING#,
+            //     },
+            // }
+
+            NCMessagePayload payload = signalingMessage.getPayload();
+            if (payload == null) {
+                // Broken message, this should not happen.
+                return;
+            }
+
+            Boolean state = payload.getState();
+            Long timestamp = payload.getTimestamp();
+
+            if (state == null || timestamp == null) {
+                // Broken message, this should not happen.
+                return;
+            }
+
+            callParticipantMessageNotifier.notifyRaiseHand(sessionId, state, timestamp);
+
+            return;
+        }
+
         // "unshareScreen" messages are directly sent to the screen peer connection when the internal signaling
         // server is used, and to the room when the external signaling server is used. However, the (relevant) data
         // of the received message ("from" and "type") is the same in both cases.

app/src/main/res/drawable/ic_hand_back_left.xml

@@ -0,0 +1,25 @@
+<!--
+    @author Google LLC
+    Copyright (C) 2023 Google LLC
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<vector xmlns:android="http://schemas.android.com/apk/res/android"
+    android:width="24dp"
+    android:height="24dp"
+    android:viewportWidth="24"
+    android:viewportHeight="24">
+    <path
+        android:fillColor="#FFFFFF"
+        android:pathData="M3 16V5.75C3 5.06 3.56 4.5 4.25 4.5S5.5 5.06 5.5 5.75V12H6.5V2.75C6.5 2.06 7.06 1.5 7.75 1.5C8.44 1.5 9 2.06 9 2.75V12H10V1.25C10 .56 10.56 0 11.25 0S12.5 .56 12.5 1.25V12H13.5V3.25C13.5 2.56 14.06 2 14.75 2S16 2.56 16 3.25V15H16.75L18.16 11.47C18.38 10.92 18.84 10.5 19.4 10.31L20.19 10.05C21 9.79 21.74 10.58 21.43 11.37L18.4 19C17.19 22 14.26 24 11 24C6.58 24 3 20.42 3 16Z" />
+</vector>

app/src/main/res/layout/call_item.xml

@@ -23,7 +23,6 @@
   -->
 
 <RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
-    xmlns:app="http://schemas.android.com/apk/res-auto"
     xmlns:tools="http://schemas.android.com/tools"
     android:id="@+id/relative_layout"
     android:layout_width="match_parent"
@@ -44,11 +43,16 @@
         android:layout_height="match_parent"
         android:visibility="invisible" />
 
+    <LinearLayout
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:orientation="horizontal"
+        android:layout_alignParentBottom="true">
+
         <TextView
             android:id="@+id/peer_nick_text_view"
             android:layout_width="wrap_content"
             android:layout_height="wrap_content"
-        android:layout_alignParentBottom="true"
             android:layout_marginStart="10dp"
             android:layout_marginBottom="6dp"
             android:ellipsize="end"
@@ -62,15 +66,25 @@
             android:id="@+id/remote_audio_off"
             android:layout_width="16dp"
             android:layout_height="16dp"
-        android:layout_alignParentBottom="true"
             android:layout_marginStart="10dp"
             android:layout_marginBottom="6dp"
-        android:layout_toEndOf="@id/peer_nick_text_view"
             android:contentDescription="@string/nc_remote_audio_off"
             android:src="@drawable/ic_mic_off_white_24px"
             android:visibility="invisible"
             tools:visibility="visible" />
 
+        <ImageView
+            android:id="@+id/raised_hand"
+            android:layout_width="16dp"
+            android:layout_height="16dp"
+            android:layout_marginStart="10dp"
+            android:layout_marginBottom="6dp"
+            android:contentDescription="@string/nc_remote_audio_off"
+            android:src="@drawable/ic_hand_back_left"
+            android:visibility="invisible"
+            tools:visibility="visible" />
+    </LinearLayout>
+
     <ProgressBar
         android:id="@+id/participant_progress_bar"
         style="?android:attr/progressBarStyle"

app/src/main/res/values/strings.xml

@@ -222,6 +222,7 @@
     <string name="nc_call_button_content_description_answer_voice_only">Answer as voice call only</string>
     <string name="nc_call_button_content_description_answer_video_call">Answer as video call</string>
     <string name="nc_call_button_content_description_switch_to_self_vide">Switch to self video</string>
+    <string name="nc_call_raised_hand">%1$s raised the hand</string>
 
     <!-- Picture in Picture -->
     <string name="nc_pip_microphone_mute">Mute microphone</string>

app/src/test/java/com/nextcloud/talk/call/CallParticipantModelTest.kt

@@ -0,0 +1,58 @@
+/*
+ * Nextcloud Talk application
+ *
+ * @author Daniel Calviño Sánchez
+ * Copyright (C) 2022 Daniel Calviño Sánchez <danxuliu@gmail.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package com.nextcloud.talk.call
+
+import org.junit.Before
+import org.junit.Test
+import org.mockito.Mockito
+
+class CallParticipantModelTest {
+    private var callParticipantModel: MutableCallParticipantModel? = null
+    private var mockedCallParticipantModelObserver: CallParticipantModel.Observer? = null
+
+    @Before
+    fun setUp() {
+        callParticipantModel = MutableCallParticipantModel("theSessionId")
+        mockedCallParticipantModelObserver = Mockito.mock(CallParticipantModel.Observer::class.java)
+    }
+
+    @Test
+    fun testSetRaisedHand() {
+        callParticipantModel!!.addObserver(mockedCallParticipantModelObserver)
+        callParticipantModel!!.setRaisedHand(true, 4815162342L)
+        Mockito.verify(mockedCallParticipantModelObserver, Mockito.only())?.onChange()
+    }
+
+    @Test
+    fun testSetRaisedHandTwice() {
+        callParticipantModel!!.addObserver(mockedCallParticipantModelObserver)
+        callParticipantModel!!.setRaisedHand(true, 4815162342L)
+        callParticipantModel!!.setRaisedHand(false, 4815162342108L)
+        Mockito.verify(mockedCallParticipantModelObserver, Mockito.times(2))?.onChange()
+    }
+
+    @Test
+    fun testSetRaisedHandTwiceWithSameValue() {
+        callParticipantModel!!.addObserver(mockedCallParticipantModelObserver)
+        callParticipantModel!!.setRaisedHand(true, 4815162342L)
+        callParticipantModel!!.setRaisedHand(true, 4815162342L)
+        Mockito.verify(mockedCallParticipantModelObserver, Mockito.only())?.onChange()
+    }
+}

app/src/test/java/com/nextcloud/talk/signaling/SignalingMessageReceiverCallParticipantTest.java

@@ -19,6 +19,7 @@
  */
 package com.nextcloud.talk.signaling;
 
+import com.nextcloud.talk.models.json.signaling.NCMessagePayload;
 import com.nextcloud.talk.models.json.signaling.NCSignalingMessage;
 
 import org.junit.Assert;
@@ -62,6 +63,27 @@ public class SignalingMessageReceiverCallParticipantTest {
         });
     }
 
+    @Test
+    public void testCallParticipantMessageRaiseHand() {
+        SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener =
+            mock(SignalingMessageReceiver.CallParticipantMessageListener.class);
+
+        signalingMessageReceiver.addListener(mockedCallParticipantMessageListener, "theSessionId");
+
+        NCSignalingMessage signalingMessage = new NCSignalingMessage();
+        signalingMessage.setFrom("theSessionId");
+        signalingMessage.setType("raiseHand");
+        signalingMessage.setRoomType("theRoomType");
+        NCMessagePayload messagePayload = new NCMessagePayload();
+        messagePayload.setType("raiseHand");
+        messagePayload.setState(Boolean.TRUE);
+        messagePayload.setTimestamp(4815162342L);
+        signalingMessage.setPayload(messagePayload);
+        signalingMessageReceiver.processSignalingMessage(signalingMessage);
+
+        verify(mockedCallParticipantMessageListener, only()).onRaiseHand(true, 4815162342L);
+    }
+
     @Test
     public void testCallParticipantMessageUnshareScreen() {
         SignalingMessageReceiver.CallParticipantMessageListener mockedCallParticipantMessageListener =