From de538437d164efafda6e15a3a9f00c5bf1f14558 Mon Sep 17 00:00:00 2001 From: Mario Danic Date: Tue, 14 Nov 2017 11:12:31 +0100 Subject: [PATCH] Update Signed-off-by: Mario Danic --- .../talk/activities/CallActivity.java | 22 ++-- .../talk/events/MediaStreamEvent.java | 34 ++++++ ....java => SessionDescriptionSendEvent.java} | 6 +- .../webrtc/MagicPeerConnectionObserver.java | 5 +- .../talk/webrtc/PeerConnectionWrapper.java | 100 +++++++++++++----- 5 files changed, 128 insertions(+), 39 deletions(-) create mode 100644 app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java rename app/src/main/java/com/nextcloud/talk/events/{SessionDescriptionSend.java => SessionDescriptionSendEvent.java} (84%) diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 42db6c04c..bdf7a32bf 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -48,7 +48,7 @@ import com.nextcloud.talk.api.models.json.signaling.NCSignalingMessage; import com.nextcloud.talk.api.models.json.signaling.Signaling; import com.nextcloud.talk.api.models.json.signaling.SignalingOverall; import com.nextcloud.talk.application.NextcloudTalkApplication; -import com.nextcloud.talk.events.SessionDescriptionSend; +import com.nextcloud.talk.events.SessionDescriptionSendEvent; import com.nextcloud.talk.persistence.entities.UserEntity; import com.nextcloud.talk.webrtc.MagicPeerConnectionObserver; import com.nextcloud.talk.webrtc.MagicSdpObserver; @@ -423,21 +423,24 @@ public class CallActivity extends AppCompatActivity { PeerConnectionWrapper peerConnectionWrapper = alwaysGetPeerConnectionWrapperForSessionId (ncSignalingMessage.getFrom(), ncSignalingMessage.getFrom().equals(callSession)); + Log.d("MARIO_RECEIVING", ncSignalingMessage.getType()); switch (ncSignalingMessage.getType()) { case "offer": case "answer": peerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick()); - peerConnectionWrapper.getPeerConnection().setLocalDescription(new MagicSdpObserver(), - new SessionDescription(SessionDescription.Type.valueOf(ncSignalingMessage.getType() - .toUpperCase()), + peerConnectionWrapper.getPeerConnection().setRemoteDescription(new MagicSdpObserver(), + new SessionDescription(SessionDescription.Type.fromCanonicalForm(ncSignalingMessage.getType()), ncSignalingMessage .getPayload().getSdp())); + if (ncSignalingMessage.getType().equals("offer")) { + peerConnectionWrapper.sendMessage(true); + } break; case "candidate": NCIceCandidate ncIceCandidate = ncSignalingMessage.getPayload().getIceCandidate(); IceCandidate iceCandidate = new IceCandidate(ncIceCandidate.getSdpMid(), ncIceCandidate.getSdpMLineIndex(), ncIceCandidate.getCandidate()); - peerConnectionWrapper.getPeerConnection().addIceCandidate(iceCandidate); + peerConnectionWrapper.addCandidate(iceCandidate); break; default: break; @@ -480,10 +483,10 @@ public class CallActivity extends AppCompatActivity { for (String sessionId : newSessions) { if (getPeerConnectionWrapperForSessionId(sessionId) == null) { - if (sessionId.compareTo(callSession) > 0) { + if (sessionId.compareTo(callSession) < 0) { PeerConnectionWrapper connectionWrapper = alwaysGetPeerConnectionWrapperForSessionId(sessionId, false); - connectionWrapper.sendOffer(); + connectionWrapper.sendMessage(false); } else { Log.d(TAG, "Waiting for offer"); } @@ -537,7 +540,7 @@ public class CallActivity extends AppCompatActivity { } peerConnectionWrapper = new PeerConnectionWrapper(peerConnectionFactory, - iceServers, sdpConstraints, magicPeerConnectionObserver, sessionId, isLocalPeer); + iceServers, sdpConstraints, magicPeerConnectionObserver, sessionId, isLocalPeer, callSession); peerConnectionWrapperList.add(peerConnectionWrapper); return peerConnectionWrapper; } @@ -666,7 +669,8 @@ public class CallActivity extends AppCompatActivity { } @Subscribe(threadMode = ThreadMode.BACKGROUND) - public void onMessageEvent(SessionDescriptionSend sessionDescriptionSend) { + public void onMessageEvent(SessionDescriptionSendEvent sessionDescriptionSend) { + Log.d("MARIO SENDING", sessionDescriptionSend.getType()); String credentials = ApiHelper.getCredentials(userEntity.getUsername(), userEntity.getToken()); NCMessageWrapper ncMessageWrapper = new NCMessageWrapper(); ncMessageWrapper.setEv("message"); diff --git a/app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java b/app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java new file mode 100644 index 000000000..34b598198 --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java @@ -0,0 +1,34 @@ +/* + * Nextcloud Talk application + * + * @author Mario Danic + * Copyright (C) 2017 Mario Danic + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.nextcloud.talk.events; + +import org.webrtc.MediaStream; + +import lombok.Data; + +@Data +public class MediaStreamEvent { + private final MediaStream mediaStream; + + public MediaStreamEvent(MediaStream mediaStream) { + this.mediaStream = mediaStream; + } +} diff --git a/app/src/main/java/com/nextcloud/talk/events/SessionDescriptionSend.java b/app/src/main/java/com/nextcloud/talk/events/SessionDescriptionSendEvent.java similarity index 84% rename from app/src/main/java/com/nextcloud/talk/events/SessionDescriptionSend.java rename to app/src/main/java/com/nextcloud/talk/events/SessionDescriptionSendEvent.java index c532e2824..e0c90c2c1 100644 --- a/app/src/main/java/com/nextcloud/talk/events/SessionDescriptionSend.java +++ b/app/src/main/java/com/nextcloud/talk/events/SessionDescriptionSendEvent.java @@ -29,14 +29,14 @@ import org.webrtc.SessionDescription; import lombok.Data; @Data -public class SessionDescriptionSend { +public class SessionDescriptionSendEvent { @Nullable private final SessionDescription sessionDescription; private final String peerId; private final String type; @Nullable private final NCIceCandidate ncIceCandidate; - public SessionDescriptionSend(@Nullable SessionDescription sessionDescription, String peerId, String type, - @Nullable NCIceCandidate ncIceCandidate) { + public SessionDescriptionSendEvent(@Nullable SessionDescription sessionDescription, String peerId, String type, + @Nullable NCIceCandidate ncIceCandidate) { this.sessionDescription = sessionDescription; this.peerId = peerId; this.type = type; diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionObserver.java b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionObserver.java index 4090bce9c..62dfc500c 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionObserver.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionObserver.java @@ -20,6 +20,8 @@ package com.nextcloud.talk.webrtc; +import android.util.Log; + import org.webrtc.DataChannel; import org.webrtc.IceCandidate; import org.webrtc.MediaStream; @@ -33,7 +35,7 @@ public class MagicPeerConnectionObserver implements PeerConnection.Observer { @Override public void onSignalingChange(PeerConnection.SignalingState signalingState) { - + Log.d("MARIO", signalingState.name()); } @Override @@ -54,6 +56,7 @@ public class MagicPeerConnectionObserver implements PeerConnection.Observer { @Override public void onIceCandidate(IceCandidate iceCandidate) { + } @Override diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java index fbf6c4aea..ad5dc50d9 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java @@ -20,9 +20,10 @@ package com.nextcloud.talk.webrtc; +import android.util.Log; + import com.nextcloud.talk.api.models.json.signaling.DataChannelMessage; -import com.nextcloud.talk.api.models.json.signaling.NCIceCandidate; -import com.nextcloud.talk.events.SessionDescriptionSend; +import com.nextcloud.talk.events.SessionDescriptionSendEvent; import org.greenrobot.eventbus.EventBus; import org.webrtc.DataChannel; @@ -33,61 +34,102 @@ import org.webrtc.PeerConnectionFactory; import org.webrtc.SessionDescription; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; public class PeerConnectionWrapper { private static PeerConnection peerConnection; private String sessionId; + private String callToken; private String nick; private boolean local; private MediaConstraints mediaConstraints; private DataChannel dataChannel; + private MagicSdpObserver magicSdpObserver; + + List iceCandidates = new ArrayList<>(); public PeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory, List iceServerList, MediaConstraints mediaConstraints, MagicPeerConnectionObserver magicPeerConnectionObserver, - String sessionId, boolean isLocalPeer) { + String sessionId, boolean isLocalPeer, String callToken) { peerConnection = peerConnectionFactory.createPeerConnection(iceServerList, mediaConstraints, magicPeerConnectionObserver); this.sessionId = sessionId; this.local = isLocalPeer; this.mediaConstraints = mediaConstraints; + this.callToken = callToken; + boolean isInitiator = this.sessionId.compareTo(callToken) < 0; + + magicSdpObserver = new MagicSdpObserver() { + public void onCreateSuccess(SessionDescription sessionDescription) { + super.onCreateSuccess(sessionDescription); + peerConnection.setLocalDescription(magicSdpObserver, sessionDescription); + } + + @Override + public void onSetSuccess() { + if (isInitiator) { + // For offering peer connection we first create offer and set + // local SDP, then after receiving answer set remote SDP. + if (peerConnection.getRemoteDescription() == null) { + // We've just set our local SDP so time to send it. + EventBus.getDefault().post(new SessionDescriptionSendEvent(peerConnection.getLocalDescription(), sessionId, + "offer", null)); + } else { + // We've just set remote description, so drain remote + // and send local ICE candidates. + drainIceCandidates(); + } + } else { + // For answering peer connection we set remote SDP and then + // create answer and set local SDP. + if (peerConnection.getLocalDescription() != null) { + // We've just set our local SDP so time to send it, drain + // remote and send local ICE candidates. + EventBus.getDefault().post(new SessionDescriptionSendEvent(peerConnection.getLocalDescription(), sessionId, + "answer", null)); + drainIceCandidates(); + } else { + // We've just set remote SDP - do nothing for now - + // answer will be created soon. + } + } + } + + }; + + } + + private void drainIceCandidates() { + for (IceCandidate iceCandidate : iceCandidates) { + peerConnection.addIceCandidate(iceCandidate); + } + + iceCandidates = new ArrayList<>(); } public void addCandidate(IceCandidate iceCandidate) { if (peerConnection.getRemoteDescription() != null) { // queue - } else { peerConnection.addIceCandidate(iceCandidate); + } else { + iceCandidates.add(iceCandidate); } - - peerConnection.addIceCandidate(iceCandidate); - NCIceCandidate ncIceCandidate = new NCIceCandidate(); - ncIceCandidate.setType("candidate"); - ncIceCandidate.setSdpMid(iceCandidate.sdpMid); - ncIceCandidate.setSdpMLineIndex(iceCandidate.sdpMLineIndex); - ncIceCandidate.setCandidate(iceCandidate.sdp); - EventBus.getDefault().post(new SessionDescriptionSend(null, sessionId, "candidate", - ncIceCandidate)); } - public void sendAnswer() { + public void sendMessage(boolean isAnswer) { - MagicSdpObserver magicSdpObserver = new MagicSdpObserver() { - public void onCreateSuccess(SessionDescription sessionDescription) { - super.onCreateSuccess(sessionDescription); - peerConnection.setLocalDescription(new MagicSdpObserver(), sessionDescription); - EventBus.getDefault().post(new SessionDescriptionSend(sessionDescription, sessionId, "answer", - null)); - } - }; - - peerConnection.createAnswer(magicSdpObserver, mediaConstraints); + Log.d("MARIO", "PREPARING " + isAnswer); + if (!isAnswer) { + peerConnection.createOffer(magicSdpObserver, mediaConstraints); + } else { + peerConnection.createAnswer(magicSdpObserver, mediaConstraints); + } } private void sendChannelData(DataChannelMessage dataChannelMessage) { - ByteBuffer buffer = ByteBuffer.wrap(dataChannelMessage.toString().getBytes()); dataChannel.send(new DataChannel.Buffer(buffer, false)); } @@ -128,8 +170,14 @@ public class PeerConnectionWrapper { public void onCreateSuccess(SessionDescription sessionDescription) { super.onCreateSuccess(sessionDescription); peerConnection.setLocalDescription(new MagicSdpObserver(), sessionDescription); - EventBus.getDefault().post(new SessionDescriptionSend(sessionDescription, sessionId, "offer", null)); } + + @Override + public void onSetSuccess() { + EventBus.getDefault().post(new SessionDescriptionSendEvent(peerConnection.getLocalDescription(), sessionId, + "offer", null)); + } + }; peerConnection.createOffer(magicSdpObserver, mediaConstraints);