From ba5de1a7e0168ecbfead68a04f872b74972c6114 Mon Sep 17 00:00:00 2001 From: Mario Danic Date: Thu, 16 Nov 2017 04:15:21 +0100 Subject: [PATCH] Many improvements Signed-off-by: Mario Danic --- .../talk/activities/CallActivity.java | 149 ++++++++++++------ .../java/com/nextcloud/talk/api/NcApi.java | 7 +- .../nextcloud/talk/events/PeerReadyEvent.java | 35 ++++ .../webrtc/MagicPeerConnectionObserver.java | 2 - .../talk/webrtc/PeerConnectionWrapper.java | 32 ++-- 5 files changed, 156 insertions(+), 69 deletions(-) create mode 100644 app/src/main/java/com/nextcloud/talk/events/PeerReadyEvent.java diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 06a896dbe..87e4ee1fb 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -49,8 +49,10 @@ import com.nextcloud.talk.api.models.json.signaling.Signaling; import com.nextcloud.talk.api.models.json.signaling.SignalingOverall; import com.nextcloud.talk.application.NextcloudTalkApplication; import com.nextcloud.talk.events.MediaStreamEvent; +import com.nextcloud.talk.events.PeerReadyEvent; import com.nextcloud.talk.events.SessionDescriptionSendEvent; import com.nextcloud.talk.persistence.entities.UserEntity; +import com.nextcloud.talk.webrtc.MagicSdpObserver; import com.nextcloud.talk.webrtc.PeerConnectionWrapper; import org.apache.commons.lang3.StringEscapeUtils; @@ -268,7 +270,8 @@ public class CallActivity extends AppCompatActivity { //we already have video and audio tracks. Now create peerconnections iceServers = new ArrayList<>(); - iceServers.add(new PeerConnection.IceServer("stun:stun.nextcloud.com:443")); + //iceServers.add(new PeerConnection.IceServer("stun:stun.l.google.com:19302")); + iceServers.add(new PeerConnection.IceServer("turn:172.104.225.9:3478")); //create sdpConstraints sdpConstraints = new MediaConstraints(); @@ -306,10 +309,6 @@ public class CallActivity extends AppCompatActivity { getPeerConnection(); //creating local mediastream - MediaStream stream = peerConnectionFactory.createLocalMediaStream("NCMS"); - stream.addTrack(localAudioTrack); - stream.addTrack(localVideoTrack); - localPeer.addStream(stream); // start pinging the call ncApi.pingCall(ApiHelper.getCredentials(userEntity.getUsername(), userEntity.getToken()), @@ -422,32 +421,64 @@ public class CallActivity extends AppCompatActivity { PeerConnectionWrapper peerConnectionWrapper = alwaysGetPeerConnectionWrapperForSessionId (ncSignalingMessage.getFrom(), ncSignalingMessage.getFrom().equals(callSession)); - switch (ncSignalingMessage.getType()) { - case "offer": - case "answer": - peerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick()); - peerConnectionWrapper.getPeerConnection().setRemoteDescription(peerConnectionWrapper - .getMagicSdpObserver(), new - SessionDescription - (SessionDescription.Type - .fromCanonicalForm(ncSignalingMessage.getType()), - ncSignalingMessage - .getPayload().getSdp())); - - break; - case "candidate": - NCIceCandidate ncIceCandidate = ncSignalingMessage.getPayload().getIceCandidate(); - IceCandidate iceCandidate = new IceCandidate(ncIceCandidate.getSdpMid(), - ncIceCandidate.getSdpMLineIndex(), ncIceCandidate.getCandidate()); - peerConnectionWrapper.addCandidate(iceCandidate); - break; - case "endOfCandidates": - Log.d("MARIO", "END CANDIDATES"); - peerConnectionWrapper.drainIceCandidates(); - break; - default: - break; + String type = null; + if (ncSignalingMessage.getType() != null) { + type = ncSignalingMessage.getType(); + } else if (ncSignalingMessage.getPayload() != null && ncSignalingMessage.getPayload().getType() != + null) { + type = ncSignalingMessage.getPayload().getType(); } + + if (type != null) { + switch (type) { + case "offer": + case "answer": + peerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick()); + peerConnectionWrapper.getPeerConnection().setRemoteDescription(new MagicSdpObserver() { + + @Override + public void onSetSuccess() { + super.onSetSuccess(); + peerConnectionWrapper.getPeerConnection().createAnswer(new MagicSdpObserver() { + @Override + public void onCreateSuccess(SessionDescription sessionDescription) { + super.onCreateSuccess(sessionDescription); + peerConnectionWrapper.getPeerConnection().setLocalDescription(new MagicSdpObserver(), + sessionDescription); + } + + }, sdpConstraints); + } + }, new SessionDescription(SessionDescription.Type.fromCanonicalForm(type), + ncSignalingMessage.getPayload().getSdp())); + + + break; + case "candidate": + NCIceCandidate ncIceCandidate = ncSignalingMessage.getPayload().getIceCandidate(); + IceCandidate iceCandidate = new IceCandidate(ncIceCandidate.getSdpMid(), + ncIceCandidate.getSdpMLineIndex(), ncIceCandidate.getCandidate()); + peerConnectionWrapper.addCandidate(iceCandidate); + break; + case "endOfCandidates": + peerConnectionWrapper.drainIceCandidates(); + if (peerConnectionWrapper.getPeerConnection().getLocalDescription() != null) { + EventBus.getDefault().post(new SessionDescriptionSendEvent + (peerConnectionWrapper.getPeerConnection().getLocalDescription(), callSession, + peerConnectionWrapper.getPeerConnection().getLocalDescription().type.canonicalForm(), + null)); + } else { + peerConnectionWrapper.getPeerConnection().createAnswer(peerConnectionWrapper + .getMagicSdpObserver(), sdpConstraints); + } + peerConnectionWrapper.sendLocalCandidates(); + break; + default: + break; + } + } + } else { + Log.d(TAG, "Something went very very wrong"); } } else { Log.d(TAG, "Something went very very wrong"); @@ -489,8 +520,10 @@ public class CallActivity extends AppCompatActivity { if (sessionId.compareTo(callSession) < 0) { PeerConnectionWrapper connectionWrapper = alwaysGetPeerConnectionWrapperForSessionId(sessionId, false); - connectionWrapper.getPeerConnection().createOffer(connectionWrapper.getMagicSdpObserver(), - sdpConstraints); + if (connectionWrapper.getPeerConnection() != null) { + connectionWrapper.getPeerConnection().createOffer(connectionWrapper.getMagicSdpObserver(), + sdpConstraints); + } } else { Log.d(TAG, "Waiting for offer"); } @@ -500,21 +533,23 @@ public class CallActivity extends AppCompatActivity { for (String sessionId : leftSessions) { if ((peerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) { - peerConnectionWrapper.getPeerConnection().close(); + if (peerConnectionWrapper.getPeerConnection() != null) { + peerConnectionWrapper.getPeerConnection().close(); + } peerConnectionWrapperList.remove(peerConnectionWrapper); } } } + private PeerConnectionWrapper alwaysGetPeerConnectionWrapperForSessionId(String sessionId, boolean isLocalPeer) { PeerConnectionWrapper peerConnectionWrapper; if ((peerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) { return peerConnectionWrapper; } else { - - peerConnectionWrapper = new PeerConnectionWrapper(peerConnectionFactory, - iceServers, sdpConstraints, sessionId, isLocalPeer, callSession); + peerConnectionWrapper = new PeerConnectionWrapper(peerConnectionFactory, + iceServers, sdpConstraints, sessionId, isLocalPeer, callSession); peerConnectionWrapperList.add(peerConnectionWrapper); return peerConnectionWrapper; } @@ -628,6 +663,16 @@ public class CallActivity extends AppCompatActivity { eventBus.unregister(this); } + @Subscribe(threadMode = ThreadMode.BACKGROUND) + public void onMessageEvent(PeerReadyEvent peerReadyEvent) { + MediaStream stream = peerConnectionFactory.createLocalMediaStream("NCMS"); + stream.addTrack(localAudioTrack); + stream.addTrack(localVideoTrack); + if (peerReadyEvent.getPeerConnection() != null) { + peerReadyEvent.getPeerConnection().addStream(stream); + } + } + @Subscribe(threadMode = ThreadMode.BACKGROUND) public void onMessageEvent(MediaStreamEvent mediaStreamEvent) { gotRemoteStream(mediaStreamEvent.getMediaStream()); @@ -660,38 +705,47 @@ public class CallActivity extends AppCompatActivity { StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("["); stringBuilder.append("{"); stringBuilder.append("\"fn\":\""); - stringBuilder.append(StringEscapeUtils.escapeEcmaScript(LoganSquare.serialize(ncMessageWrapper - .getSignalingMessage())) + "\""); + stringBuilder.append(StringEscapeUtils.escapeJson(LoganSquare.serialize(ncMessageWrapper + .getSignalingMessage()))).append("\""); stringBuilder.append(","); stringBuilder.append("\"sessionId\":"); - stringBuilder.append("\"").append(callSession).append("\""); + stringBuilder.append("\"").append(StringEscapeUtils.escapeJson(callSession)).append("\""); stringBuilder.append(","); stringBuilder.append("\"ev\":\"message\""); stringBuilder.append("}"); - stringBuilder.append("]"); + List strings = new ArrayList<>(); String stringToSend = stringBuilder.toString(); - Log.d("MARIO", stringToSend); + strings.add(stringToSend); + + List wrappers = new ArrayList<>(); + wrappers.add(ncMessageWrapper); ncApi.sendSignalingMessages(credentials, ApiHelper.getUrlForSignaling(userEntity.getBaseUrl()), - stringToSend) + strings.toString()) .subscribeOn(Schedulers.newThread()) - .subscribe(new Observer() { + .subscribe(new Observer() { @Override public void onSubscribe(Disposable d) { } @Override - public void onNext(GenericOverall genericOverall) { - + public void onNext(SignalingOverall signalingOverall) { + if (signalingOverall.getOcs().getSignalings() != null) { + for (int i = 0; i < signalingOverall.getOcs().getSignalings().size(); i++) { + try { + receivedSignalingMessage(signalingOverall.getOcs().getSignalings().get(i)); + } catch (IOException e) { + e.printStackTrace(); + } + } + } } @Override public void onError(Throwable e) { - } @Override @@ -701,5 +755,4 @@ public class CallActivity extends AppCompatActivity { }); } - } diff --git a/app/src/main/java/com/nextcloud/talk/api/NcApi.java b/app/src/main/java/com/nextcloud/talk/api/NcApi.java index f635c1a29..e5f247422 100644 --- a/app/src/main/java/com/nextcloud/talk/api/NcApi.java +++ b/app/src/main/java/com/nextcloud/talk/api/NcApi.java @@ -35,8 +35,8 @@ import com.nextcloud.talk.api.models.json.userprofile.UserProfileOverall; import java.util.Map; import io.reactivex.Observable; -import retrofit2.http.Body; import retrofit2.http.DELETE; +import retrofit2.http.Field; import retrofit2.http.FieldMap; import retrofit2.http.FormUrlEncoded; import retrofit2.http.GET; @@ -165,9 +165,10 @@ public interface NcApi { Server URL is: baseUrl + ocsApiVersion + spreedApiVersion + /signaling */ + @FormUrlEncoded @POST - Observable sendSignalingMessages(@Header("Authorization") String authorization, @Url String url, - @Body String message); + Observable sendSignalingMessages(@Header("Authorization") String authorization, @Url String url, + @Field("messages") String messages); /* Server URL is: baseUrl + ocsApiVersion + spreedApiVersion + /signaling diff --git a/app/src/main/java/com/nextcloud/talk/events/PeerReadyEvent.java b/app/src/main/java/com/nextcloud/talk/events/PeerReadyEvent.java new file mode 100644 index 000000000..8bcbcf7da --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/events/PeerReadyEvent.java @@ -0,0 +1,35 @@ +/* + * Nextcloud Talk application + * + * @author Mario Danic + * Copyright (C) 2017 Mario Danic + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.nextcloud.talk.events; + + +import org.webrtc.PeerConnection; + +import lombok.Data; + +@Data +public class PeerReadyEvent { + private final PeerConnection peerConnection; + + public PeerReadyEvent(PeerConnection peerConnection) { + this.peerConnection = peerConnection; + } +} diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionObserver.java b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionObserver.java index 62dfc500c..940d7303b 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionObserver.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionObserver.java @@ -55,8 +55,6 @@ public class MagicPeerConnectionObserver implements PeerConnection.Observer { @Override public void onIceCandidate(IceCandidate iceCandidate) { - - } @Override diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java index 6a668d799..53e268423 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java @@ -25,6 +25,7 @@ import android.util.Log; import com.nextcloud.talk.api.models.json.signaling.DataChannelMessage; import com.nextcloud.talk.api.models.json.signaling.NCIceCandidate; import com.nextcloud.talk.events.MediaStreamEvent; +import com.nextcloud.talk.events.PeerReadyEvent; import com.nextcloud.talk.events.SessionDescriptionSendEvent; import org.greenrobot.eventbus.EventBus; @@ -79,6 +80,7 @@ public class PeerConnectionWrapper { @Override public void onAddStream(MediaStream mediaStream) { + Log.d("MARIO", "MEDIA"); EventBus.getDefault().post(new MediaStreamEvent(mediaStream)); } @@ -90,9 +92,9 @@ public class PeerConnectionWrapper { @Override public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) { - if (iceGatheringState.equals(PeerConnection.IceGatheringState.COMPLETE)) { + /*if (iceGatheringState.equals(PeerConnection.IceGatheringState.COMPLETE)) { sendLocalCandidates(); - } + }*/ } @Override @@ -106,8 +108,13 @@ public class PeerConnectionWrapper { }; - peerConnection = peerConnectionFactory.createPeerConnection(iceServerList, mediaConstraints, - magicPeerConnectionObserver); + peerConnection = peerConnectionFactory.createPeerConnection(iceServerList, mediaConstraints, + magicPeerConnectionObserver); + + + if (isLocalPeer) { + EventBus.getDefault().post(new PeerReadyEvent(peerConnection)); + } this.sessionId = sessionId; this.local = isLocalPeer; this.mediaConstraints = mediaConstraints; @@ -139,11 +146,7 @@ public class PeerConnectionWrapper { if (peerConnection.getRemoteDescription() == null) { // We've just set our local SDP so time to send it. EventBus.getDefault().post(new SessionDescriptionSendEvent(peerConnection.getLocalDescription(), sessionId, - "offer", null)); - } else { - // We've just set remote description, so drain remote - // and send local ICE candidates. - drainIceCandidates(); + peerConnection.getLocalDescription().type.canonicalForm(), null)); } } else { // For anwering peer connection we set remote SDP and then @@ -152,18 +155,13 @@ public class PeerConnectionWrapper { EventBus.getDefault().post(new SessionDescriptionSendEvent(peerConnection.getLocalDescription (), sessionId, - "answer", null)); + peerConnection.getLocalDescription().type.canonicalForm(), null)); // We've just set our local SDP so time to send it, drain // remote and send local ICE candidates. } else { - peerConnection.createAnswer(magicSdpObserver, mediaConstraints); - // We've just set remote SDP - do nothing for now - // answer will be created soon. - /*Log.d("MARIO", "CREATING ANSWER"); - drainIceCandidates(); - peerConnection.createAnswer(magicSdpObserver, mediaConstraints);*/ } } @@ -174,11 +172,13 @@ public class PeerConnectionWrapper { } - private void sendLocalCandidates() { + public void sendLocalCandidates() { for (NCIceCandidate ncIceCandidate : localCandidates) { EventBus.getDefault().post(new SessionDescriptionSendEvent(null, sessionId, "candidate", ncIceCandidate)); } + + localCandidates = new ArrayList<>(); } public void drainIceCandidates() {