diff --git a/app/src/main/java/com/nextcloud/talk/controllers/CallController.java b/app/src/main/java/com/nextcloud/talk/controllers/CallController.java index 62e4626db..e00ad1034 100644 --- a/app/src/main/java/com/nextcloud/talk/controllers/CallController.java +++ b/app/src/main/java/com/nextcloud/talk/controllers/CallController.java @@ -1280,8 +1280,10 @@ public class CallController extends BaseController { } private void processMessage(NCSignalingMessage ncSignalingMessage) { - if (ncSignalingMessage.getRoomType().equals("video")) { - MagicPeerConnectionWrapper magicPeerConnectionWrapper = alwaysGetPeerConnectionWrapperForSessionId(ncSignalingMessage.getFrom(), false); + if (ncSignalingMessage.getRoomType().equals("video") || ncSignalingMessage.getRoomType().equals("screen")) { + MagicPeerConnectionWrapper magicPeerConnectionWrapper = + getPeerConnectionWrapperForSessionIdAndType(ncSignalingMessage.getFrom(), + ncSignalingMessage.getRoomType(),false); String type = null; if (ncSignalingMessage.getPayload() != null && ncSignalingMessage.getPayload().getType() != null) { @@ -1292,6 +1294,9 @@ public class CallController extends BaseController { if (type != null) { switch (type) { + case "unshareScreen": + endPeerConnection(ncSignalingMessage.getFrom(), true); + break; case "offer": case "answer": magicPeerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick()); @@ -1344,7 +1349,7 @@ public class CallController extends BaseController { } for (int i = 0; i < magicPeerConnectionWrapperList.size(); i++) { - endPeerConnection(magicPeerConnectionWrapperList.get(i).getSessionId()); + endPeerConnection(magicPeerConnectionWrapperList.get(i).getSessionId(), false); } @@ -1500,11 +1505,11 @@ public class CallController extends BaseController { hasMCU = hasExternalSignalingServer && webSocketClient != null && webSocketClient.hasMCU(); for (String sessionId : newSessions) { - alwaysGetPeerConnectionWrapperForSessionId(sessionId, hasMCU && sessionId.equals(webSocketClient.getSessionId())); + getPeerConnectionWrapperForSessionIdAndType(sessionId, "video",hasMCU && sessionId.equals(webSocketClient.getSessionId())); } for (String sessionId : oldSesssions) { - endPeerConnection(sessionId); + endPeerConnection(sessionId, false); } } @@ -1545,20 +1550,35 @@ public class CallController extends BaseController { magicPeerConnectionWrapperList.remove(magicPeerConnectionWrapper); } - private MagicPeerConnectionWrapper alwaysGetPeerConnectionWrapperForSessionId(String sessionId, boolean publisher) { + private MagicPeerConnectionWrapper getPeerConnectionWrapperForSessionId(String sessionId, String type) { + for (int i = 0; i < magicPeerConnectionWrapperList.size(); i++) { + if (magicPeerConnectionWrapperList.get(i).getSessionId().equals(sessionId) && magicPeerConnectionWrapperList.get(i).getVideoStreamType().equals(type)) { + return magicPeerConnectionWrapperList.get(i); + } + } + + return null; + } + + private MagicPeerConnectionWrapper getPeerConnectionWrapperForSessionIdAndType(String sessionId, String type, boolean publisher) { MagicPeerConnectionWrapper magicPeerConnectionWrapper; - if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) { + if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId, type)) != null) { return magicPeerConnectionWrapper; } else { if (hasMCU && publisher) { magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory, - iceServers, sdpConstraintsForMCU, sessionId, callSession, localMediaStream, true, true); + iceServers, sdpConstraintsForMCU, sessionId, callSession, localMediaStream, true, true, type); } else if (hasMCU) { magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory, - iceServers, sdpConstraints, sessionId, callSession, null, false, true); + iceServers, sdpConstraints, sessionId, callSession, null, false, true, type); } else { - magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory, - iceServers, sdpConstraints, sessionId, callSession, localMediaStream, false, false); + if (!type.equals("screen")) { + magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory, + iceServers, sdpConstraints, sessionId, callSession, localMediaStream, false, false, type); + } else { + magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory, + iceServers, sdpConstraints, sessionId, callSession, null, false, false, type); + } } magicPeerConnectionWrapperList.add(magicPeerConnectionWrapper); @@ -1566,21 +1586,38 @@ public class CallController extends BaseController { } } - private MagicPeerConnectionWrapper getPeerConnectionWrapperForSessionId(String sessionId) { + private List getPeerConnectionWrapperListForSessionId(String sessionId) { + List internalList = new ArrayList<>(); for (MagicPeerConnectionWrapper magicPeerConnectionWrapper : magicPeerConnectionWrapperList) { if (magicPeerConnectionWrapper.getSessionId().equals(sessionId)) { - return magicPeerConnectionWrapper; + internalList.add(magicPeerConnectionWrapper); } } - return null; + + return internalList; } - private void endPeerConnection(String sessionId) { + private void endPeerConnection(String sessionId, boolean justScreen) { + List magicPeerConnectionWrappers; MagicPeerConnectionWrapper magicPeerConnectionWrapper; - if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null && getActivity() + if (!(magicPeerConnectionWrappers = getPeerConnectionWrapperListForSessionId(sessionId)).isEmpty() && getActivity() != null) { - getActivity().runOnUiThread(() -> removeMediaStream(sessionId)); - deleteMagicPeerConnection(magicPeerConnectionWrapper); + for (int i = 0; i < magicPeerConnectionWrappers.size(); i++) { + magicPeerConnectionWrapper = magicPeerConnectionWrappers.get(i); + if (magicPeerConnectionWrapper.getSessionId().equals(sessionId)) { + if (magicPeerConnectionWrapper.getVideoStreamType().equals("screen")) { + MagicPeerConnectionWrapper finalMagicPeerConnectionWrapper = magicPeerConnectionWrapper; + getActivity().runOnUiThread(() -> removeMediaStream(sessionId + "+" + + finalMagicPeerConnectionWrapper.getVideoStreamType())); + deleteMagicPeerConnection(magicPeerConnectionWrapper); + } else if (!justScreen) { + MagicPeerConnectionWrapper finalMagicPeerConnectionWrapper = magicPeerConnectionWrapper; + getActivity().runOnUiThread(() -> removeMediaStream(sessionId + "+" + + finalMagicPeerConnectionWrapper.getVideoStreamType())); + deleteMagicPeerConnection(magicPeerConnectionWrapper); + } + } + } } } @@ -1613,7 +1650,7 @@ public class CallController extends BaseController { public void onMessageEvent(PeerConnectionEvent peerConnectionEvent) { if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent.PeerConnectionEventType .PEER_CLOSED)) { - endPeerConnection(peerConnectionEvent.getSessionId()); + endPeerConnection(peerConnectionEvent.getSessionId(), false); } else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent .PeerConnectionEventType.SENSOR_FAR) || peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent @@ -1630,14 +1667,14 @@ public class CallController extends BaseController { } } else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent .PeerConnectionEventType.NICK_CHANGE)) { - gotNick(peerConnectionEvent.getSessionId(), peerConnectionEvent.getNick(), true); + gotNick(peerConnectionEvent.getSessionId() + "+" + peerConnectionEvent.getVideoStreamType(), peerConnectionEvent.getNick(), true); } else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent .PeerConnectionEventType.VIDEO_CHANGE) && !isVoiceOnlyCall) { - gotAudioOrVideoChange(true, peerConnectionEvent.getSessionId(), + gotAudioOrVideoChange(true, peerConnectionEvent.getSessionId() + "+" + peerConnectionEvent.getVideoStreamType(), peerConnectionEvent.getChangeValue()); } else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent .PeerConnectionEventType.AUDIO_CHANGE)) { - gotAudioOrVideoChange(false, peerConnectionEvent.getSessionId(), + gotAudioOrVideoChange(false, peerConnectionEvent.getSessionId() + "+" + peerConnectionEvent.getVideoStreamType(), peerConnectionEvent.getChangeValue()); } } @@ -1668,9 +1705,9 @@ public class CallController extends BaseController { if (mediaStreamEvent.getMediaStream() != null) { setupVideoStreamForLayout(mediaStreamEvent.getMediaStream(), mediaStreamEvent.getSession(), mediaStreamEvent.getMediaStream().videoTracks != null - && mediaStreamEvent.getMediaStream().videoTracks.size() > 0); + && mediaStreamEvent.getMediaStream().videoTracks.size() > 0, mediaStreamEvent.getVideoStreamType()); } else { - setupVideoStreamForLayout(null, mediaStreamEvent.getSession(), false); + setupVideoStreamForLayout(null, mediaStreamEvent.getSession(), false, mediaStreamEvent.getVideoStreamType()); } } @@ -1681,7 +1718,7 @@ public class CallController extends BaseController { ncMessageWrapper.setSessionId(callSession); NCSignalingMessage ncSignalingMessage = new NCSignalingMessage(); ncSignalingMessage.setTo(sessionDescriptionSend.getPeerId()); - ncSignalingMessage.setRoomType("video"); + ncSignalingMessage.setRoomType(sessionDescriptionSend.getVideoStreamType()); ncSignalingMessage.setType(sessionDescriptionSend.getType()); NCMessagePayload ncMessagePayload = new NCMessagePayload(); ncMessagePayload.setType(sessionDescriptionSend.getType()); @@ -1768,7 +1805,7 @@ public class CallController extends BaseController { private void setupAvatarForSession(String session) { if (remoteRenderersLayout != null) { - RelativeLayout relativeLayout = remoteRenderersLayout.findViewWithTag(session); + RelativeLayout relativeLayout = remoteRenderersLayout.findViewWithTag(session + "+video"); if (relativeLayout != null) { ImageView avatarImageView = relativeLayout.findViewById(R.id.avatarImageView); @@ -1791,14 +1828,14 @@ public class CallController extends BaseController { } } - private void setupVideoStreamForLayout(@Nullable MediaStream mediaStream, String session, boolean enable) { + private void setupVideoStreamForLayout(@Nullable MediaStream mediaStream, String session, boolean enable, String videoStreamType) { boolean isInitialLayoutSetupForPeer = false; if (remoteRenderersLayout.findViewWithTag(session) == null) { - setupNewPeerLayout(session); + setupNewPeerLayout(session, videoStreamType); isInitialLayoutSetupForPeer = true; } - RelativeLayout relativeLayout = remoteRenderersLayout.findViewWithTag(session); + RelativeLayout relativeLayout = remoteRenderersLayout.findViewWithTag(session + "+" + videoStreamType); SurfaceViewRenderer surfaceViewRenderer = relativeLayout.findViewById(R.id.surface_view); ImageView imageView = relativeLayout.findViewById(R.id.avatarImageView); @@ -1850,13 +1887,13 @@ public class CallController extends BaseController { } } - private void setupNewPeerLayout(String session) { - if (remoteRenderersLayout.findViewWithTag(session) == null && getActivity() != null) { + private void setupNewPeerLayout(String session, String type) { + if (remoteRenderersLayout.findViewWithTag(session + "+" + type) == null && getActivity() != null) { getActivity().runOnUiThread(() -> { RelativeLayout relativeLayout = (RelativeLayout) getActivity().getLayoutInflater().inflate(R.layout.call_item, remoteRenderersLayout, false); - relativeLayout.setTag(session); + relativeLayout.setTag(session + "+" + type); SurfaceViewRenderer surfaceViewRenderer = relativeLayout.findViewById(R.id .surface_view); @@ -1869,12 +1906,14 @@ public class CallController extends BaseController { surfaceViewRenderer.setOnClickListener(videoOnClickListener); remoteRenderersLayout.addView(relativeLayout); if (hasExternalSignalingServer) { - gotNick(session, webSocketClient.getDisplayNameForSession(session), false); + gotNick(session + "+" + type, webSocketClient.getDisplayNameForSession(session), false); } else { - gotNick(session, getPeerConnectionWrapperForSessionId(session).getNick(), false); + gotNick(session + "+" + type, getPeerConnectionWrapperForSessionIdAndType(session, "video", false).getNick(), false); } - setupAvatarForSession(session); + if (type.equals("video")) { + setupAvatarForSession(session); + } callControls.setZ(100.0f); }); diff --git a/app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java b/app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java index a05f0c2b2..e5a2ba67e 100644 --- a/app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java +++ b/app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java @@ -29,9 +29,11 @@ import lombok.Data; public class MediaStreamEvent { private final MediaStream mediaStream; private final String session; + private final String videoStreamType; - public MediaStreamEvent(@Nullable MediaStream mediaStream, String session) { + public MediaStreamEvent(@Nullable MediaStream mediaStream, String session, String videoStreamType) { this.mediaStream = mediaStream; this.session = session; + this.videoStreamType = videoStreamType; } } diff --git a/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java b/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java index ef2a42201..f85954c4a 100644 --- a/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java +++ b/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java @@ -29,13 +29,15 @@ public class PeerConnectionEvent { private final String sessionId; private final String nick; private final Boolean changeValue; + private final String videoStreamType; public PeerConnectionEvent(PeerConnectionEventType peerConnectionEventType, @Nullable String sessionId, - @Nullable String nick, Boolean changeValue) { + @Nullable String nick, Boolean changeValue, @Nullable String videoStreamType) { this.peerConnectionEventType = peerConnectionEventType; this.nick = nick; this.changeValue = changeValue; this.sessionId = sessionId; + this.videoStreamType = videoStreamType; } public enum PeerConnectionEventType { diff --git a/app/src/main/java/com/nextcloud/talk/events/SessionDescriptionSendEvent.java b/app/src/main/java/com/nextcloud/talk/events/SessionDescriptionSendEvent.java index 204e73a9e..5925b47ba 100644 --- a/app/src/main/java/com/nextcloud/talk/events/SessionDescriptionSendEvent.java +++ b/app/src/main/java/com/nextcloud/talk/events/SessionDescriptionSendEvent.java @@ -35,12 +35,14 @@ public class SessionDescriptionSendEvent { private final String type; @Nullable private final NCIceCandidate ncIceCandidate; + private final String videoStreamType; public SessionDescriptionSendEvent(@Nullable SessionDescription sessionDescription, String peerId, String type, - @Nullable NCIceCandidate ncIceCandidate) { + @Nullable NCIceCandidate ncIceCandidate, @Nullable String videoStreamType) { this.sessionDescription = sessionDescription; this.peerId = peerId; this.type = type; this.ncIceCandidate = ncIceCandidate; + this.videoStreamType = videoStreamType; } } diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/MagicAudioManager.java b/app/src/main/java/com/nextcloud/talk/webrtc/MagicAudioManager.java index 17ca13563..3fe9fed5b 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/MagicAudioManager.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/MagicAudioManager.java @@ -176,7 +176,7 @@ public class MagicAudioManager { setAudioDeviceInternal(MagicAudioManager.AudioDevice.EARPIECE); EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .SENSOR_NEAR, null, null, null)); + .SENSOR_NEAR, null, null, null, null)); } else { // Sensor reports that a "handset is removed from a person's ear", or @@ -184,7 +184,7 @@ public class MagicAudioManager { setAudioDeviceInternal(MagicAudioManager.AudioDevice.SPEAKER_PHONE); EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .SENSOR_FAR, null, null, null)); + .SENSOR_FAR, null, null, null, null)); } } } diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java index 82d0f4791..38ca9efe0 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java @@ -71,14 +71,16 @@ public class MagicPeerConnectionWrapper { private MediaStream localMediaStream; private boolean isMCUPublisher; + private String videoStreamType; public MagicPeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory, List iceServerList, MediaConstraints mediaConstraints, String sessionId, String localSession, @Nullable MediaStream mediaStream, - boolean isMCUPublisher, boolean hasMCU) { + boolean isMCUPublisher, boolean hasMCU, String videoStreamType) { this.localMediaStream = mediaStream; + this.videoStreamType = videoStreamType; this.sessionId = sessionId; this.mediaConstraints = mediaConstraints; @@ -114,6 +116,10 @@ public class MagicPeerConnectionWrapper { } } + public String getVideoStreamType() { + return videoStreamType; + } + public void removePeerConnection() { if (magicDataChannel != null) { magicDataChannel.dispose(); @@ -243,32 +249,32 @@ public class MagicPeerConnectionWrapper { if (!internalNick.equals(nick)) { setNick(nick); EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .NICK_CHANGE, sessionId, getNick(), null)); + .NICK_CHANGE, sessionId, getNick(), null, videoStreamType)); } } else { if (dataChannelMessage.getPayload() != null) { HashMap payloadHashMap = (HashMap) dataChannelMessage.getPayload(); EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .NICK_CHANGE, payloadHashMap.get("userid"), payloadHashMap.get("name"), null)); + .NICK_CHANGE, payloadHashMap.get("userid"), payloadHashMap.get("name"), null, videoStreamType)); } } } else if ("audioOn".equals(dataChannelMessage.getType())) { remoteAudioOn = true; EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .AUDIO_CHANGE, sessionId, null, remoteAudioOn)); + .AUDIO_CHANGE, sessionId, null, remoteAudioOn, videoStreamType)); } else if ("audioOff".equals(dataChannelMessage.getType())) { remoteAudioOn = false; EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .AUDIO_CHANGE, sessionId, null, remoteAudioOn)); + .AUDIO_CHANGE, sessionId, null, remoteAudioOn, videoStreamType)); } else if ("videoOn".equals(dataChannelMessage.getType())) { remoteVideoOn = true; EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .VIDEO_CHANGE, sessionId, null, remoteVideoOn)); + .VIDEO_CHANGE, sessionId, null, remoteVideoOn, videoStreamType)); } else if ("videoOff".equals(dataChannelMessage.getType())) { remoteVideoOn = false; EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .VIDEO_CHANGE, sessionId, null, remoteVideoOn)); + .VIDEO_CHANGE, sessionId, null, remoteVideoOn, videoStreamType)); } } catch (IOException e) { Log.d(TAG, "Failed to parse data channel message"); @@ -290,7 +296,7 @@ public class MagicPeerConnectionWrapper { .PEER_CONNECTED, sessionId, null, null));*/ if (!isMCUPublisher) { - EventBus.getDefault().post(new MediaStreamEvent(remoteMediaStream, sessionId)); + EventBus.getDefault().post(new MediaStreamEvent(remoteMediaStream, sessionId, videoStreamType)); } if (hasInitiated) { @@ -299,7 +305,7 @@ public class MagicPeerConnectionWrapper { } else if (iceConnectionState.equals(PeerConnection.IceConnectionState.CLOSED)) { EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType - .PEER_CLOSED, sessionId, null, null)); + .PEER_CLOSED, sessionId, null, null, videoStreamType)); } } @@ -320,7 +326,7 @@ public class MagicPeerConnectionWrapper { ncIceCandidate.setSdpMLineIndex(iceCandidate.sdpMLineIndex); ncIceCandidate.setCandidate(iceCandidate.sdp); EventBus.getDefault().post(new SessionDescriptionSendEvent(null, sessionId, - "candidate", ncIceCandidate)); + "candidate", ncIceCandidate, videoStreamType)); } @Override @@ -336,7 +342,7 @@ public class MagicPeerConnectionWrapper { @Override public void onRemoveStream(MediaStream mediaStream) { if (!isMCUPublisher) { - EventBus.getDefault().post(new MediaStreamEvent(null, sessionId)); + EventBus.getDefault().post(new MediaStreamEvent(null, sessionId, videoStreamType)); } } @@ -383,7 +389,7 @@ public class MagicPeerConnectionWrapper { EventBus.getDefault().post(new SessionDescriptionSendEvent(sessionDescriptionWithPreferredCodec, sessionId, - sessionDescription.type.canonicalForm().toLowerCase(), null)); + sessionDescription.type.canonicalForm().toLowerCase(), null, videoStreamType)); if (peerConnection != null) { peerConnection.setLocalDescription(magicSdpObserver, sessionDescriptionWithPreferredCodec); diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebRTCUtils.java b/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebRTCUtils.java index 219224925..e0e866a82 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebRTCUtils.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/MagicWebRTCUtils.java @@ -129,7 +129,7 @@ public class MagicWebRTCUtils { } Log.d(TAG, "Change media description from: " + lines[mLineIndex] + " to " + newMLine); lines[mLineIndex] = newMLine; - return joinString(Arrays.asList(lines), "\r\n", true /* delimiterAtEnd */); + return joinString(Arrays.asList(lines), "\r\n", true); } /**