From 26eb025c2856889da20cedfac55be6d1f8a9853b Mon Sep 17 00:00:00 2001 From: Mario Danic Date: Sun, 3 Dec 2017 15:18:15 +0100 Subject: [PATCH] Done Signed-off-by: Mario Danic --- .../talk/activities/CallActivity.java | 89 +++++++++++-------- .../talk/events/MediaStreamEvent.java | 4 +- .../talk/events/PeerConnectionEvent.java | 32 +++++++ .../talk/jobs/AccountRemovalJob.java | 30 ++++++- .../com/nextcloud/talk/utils/PushUtils.java | 13 ++- .../webrtc/MagicPeerConnectionWrapper.java | 45 ++++++---- app/src/main/res/layout/activity_call.xml | 5 +- 7 files changed, 158 insertions(+), 60 deletions(-) create mode 100644 app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java index 8ec715f28..698cfde82 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.java @@ -50,6 +50,7 @@ import com.nextcloud.talk.api.models.json.signaling.Signaling; import com.nextcloud.talk.api.models.json.signaling.SignalingOverall; import com.nextcloud.talk.application.NextcloudTalkApplication; import com.nextcloud.talk.events.MediaStreamEvent; +import com.nextcloud.talk.events.PeerConnectionEvent; import com.nextcloud.talk.events.SessionDescriptionSendEvent; import com.nextcloud.talk.persistence.entities.UserEntity; import com.nextcloud.talk.webrtc.MagicAudioManager; @@ -321,7 +322,7 @@ public class CallActivity extends AppCompatActivity { @Override public void onNext(GenericOverall genericOverall) { callSession = callOverall.getOcs().getData().getSessionId(); - localPeer = alwaysGetPeerConnectionWrapperForSessionId(callSession, true). + localPeer = alwaysGetPeerConnectionWrapperForSessionId(callSession). getPeerConnection(); // start pinging the call @@ -433,14 +434,14 @@ public class CallActivity extends AppCompatActivity { NCSignalingMessage.class); if (ncSignalingMessage.getRoomType().equals("video")) { MagicPeerConnectionWrapper magicPeerConnectionWrapper = alwaysGetPeerConnectionWrapperForSessionId - (ncSignalingMessage.getFrom(), ncSignalingMessage.getFrom().equals(callSession)); + (ncSignalingMessage.getFrom()); String type = null; - if (ncSignalingMessage.getType() != null) { - type = ncSignalingMessage.getType(); - } else if (ncSignalingMessage.getPayload() != null && ncSignalingMessage.getPayload().getType() != + if (ncSignalingMessage.getPayload() != null && ncSignalingMessage.getPayload().getType() != null) { type = ncSignalingMessage.getPayload().getType(); + } else if (ncSignalingMessage.getType() != null) { + type = ncSignalingMessage.getType(); } if (type != null) { @@ -448,14 +449,9 @@ public class CallActivity extends AppCompatActivity { case "offer": case "answer": magicPeerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick()); - if (!magicPeerConnectionWrapper.getPeerConnection().signalingState().equals - (PeerConnection.SignalingState.STABLE) && - magicPeerConnectionWrapper.getPeerConnection().getRemoteDescription() == null || - magicPeerConnectionWrapper.getPeerConnection().getLocalDescription() == null) { magicPeerConnectionWrapper.getPeerConnection().setRemoteDescription(magicPeerConnectionWrapper .getMagicSdpObserver(), new SessionDescription(SessionDescription.Type.fromCanonicalForm(type), ncSignalingMessage.getPayload().getSdp())); - } break; case "candidate": NCIceCandidate ncIceCandidate = ncSignalingMessage.getPayload().getIceCandidate(); @@ -497,6 +493,8 @@ public class CallActivity extends AppCompatActivity { } } + + for (MagicPeerConnectionWrapper magicPeerConnectionWrapper : magicPeerConnectionWrapperList) { if (!magicPeerConnectionWrapper.getSessionId().equals(callSession)) { oldSesssions.add(magicPeerConnectionWrapper.getSessionId()); @@ -514,39 +512,27 @@ public class CallActivity extends AppCompatActivity { return; } - MagicPeerConnectionWrapper magicPeerConnectionWrapper; - for (String sessionId : newSessions) { - if (getPeerConnectionWrapperForSessionId(sessionId) == null) { - if (sessionId.compareTo(callSession) < 0) { - MagicPeerConnectionWrapper connectionWrapper = alwaysGetPeerConnectionWrapperForSessionId(sessionId, - false); - if (connectionWrapper.getPeerConnection() != null) { - connectionWrapper.getPeerConnection().createOffer(connectionWrapper.getMagicSdpObserver(), - sdpConstraints); - } - } else { - Log.d(TAG, "Waiting for offer"); - } - - } + alwaysGetPeerConnectionWrapperForSessionId(sessionId); } for (String sessionId : leftSessions) { - if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) { - if (magicPeerConnectionWrapper.getPeerConnection() != null) { - magicPeerConnectionWrapper.getPeerConnection().close(); - } - magicPeerConnectionWrapperList.remove(magicPeerConnectionWrapper); - } + endPeerConnection(sessionId); } } - private MagicPeerConnectionWrapper alwaysGetPeerConnectionWrapperForSessionId(String sessionId, boolean isLocalPeer) { + private void deleteMagicPeerConnection(MagicPeerConnectionWrapper magicPeerConnectionWrapper) { + if (magicPeerConnectionWrapper.getPeerConnection() != null) { + magicPeerConnectionWrapper.getPeerConnection().close(); + } + magicPeerConnectionWrapperList.remove(magicPeerConnectionWrapper); + } + + private MagicPeerConnectionWrapper alwaysGetPeerConnectionWrapperForSessionId(String sessionId) { MagicPeerConnectionWrapper magicPeerConnectionWrapper; - if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) { - return magicPeerConnectionWrapper; + if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) { + return magicPeerConnectionWrapper; } else { magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory, iceServers, sdpConstraints, sessionId, callSession); @@ -612,6 +598,7 @@ public class CallActivity extends AppCompatActivity { private void gotRemoteStream(MediaStream stream, String session) { //we have remote video stream. add to the renderer. + removeMediaStream(session); if (stream.videoTracks.size() < 2 && stream.audioTracks.size() < 2) { if (stream.videoTracks.size() == 1) { @@ -631,7 +618,7 @@ public class CallActivity extends AppCompatActivity { surfaceViewRenderer.init(rootEglBase.getEglBaseContext(), null); surfaceViewRenderer.setZOrderMediaOverlay(true); surfaceViewRenderer.setEnableHardwareScaler(true); - surfaceViewRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL); + surfaceViewRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT); VideoRenderer remoteRenderer = new VideoRenderer(surfaceViewRenderer); videoRendererHashMap.put(session, remoteRenderer); videoTrack.addRenderer(remoteRenderer); @@ -688,9 +675,38 @@ public class CallActivity extends AppCompatActivity { eventBus.unregister(this); } + @Subscribe(threadMode = ThreadMode.BACKGROUND) + public void onMessageEvent(PeerConnectionEvent peerConnectionEvent) { + endPeerConnection(peerConnectionEvent.getSessionId()); + } + + private void endPeerConnection(String sessionId) { + MagicPeerConnectionWrapper magicPeerConnectionWrapper; + if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) { + runOnUiThread(() -> removeMediaStream(sessionId)); + deleteMagicPeerConnection(magicPeerConnectionWrapper); + } + } + + private void removeMediaStream(String sessionId) { + if (remoteRenderersLayout.getChildCount() > 0) { + for (int i = 0; i < remoteRenderersLayout.getChildCount(); i++) { + if (remoteRenderersLayout.getChildAt(i).getTag().equals(sessionId)) { + remoteRenderersLayout.removeViewAt(i); + remoteRenderersLayout.invalidate(); + break; + } + } + } + } + @Subscribe(threadMode = ThreadMode.BACKGROUND) public void onMessageEvent(MediaStreamEvent mediaStreamEvent) { - gotRemoteStream(mediaStreamEvent.getMediaStream(), mediaStreamEvent.getSession()); + if (mediaStreamEvent.getMediaStream() != null) { + gotRemoteStream(mediaStreamEvent.getMediaStream(), mediaStreamEvent.getSession()); + } else { + removeMediaStream(mediaStreamEvent.getSession()); + } } @Subscribe(threadMode = ThreadMode.BACKGROUND) @@ -700,6 +716,7 @@ public class CallActivity extends AppCompatActivity { ncMessageWrapper.setEv("message"); ncMessageWrapper.setSessionId(callSession); NCSignalingMessage ncSignalingMessage = new NCSignalingMessage(); + ncSignalingMessage.setFrom(callSession); ncSignalingMessage.setTo(sessionDescriptionSend.getPeerId()); ncSignalingMessage.setRoomType("video"); ncSignalingMessage.setType(sessionDescriptionSend.getType()); diff --git a/app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java b/app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java index 8fd8f5762..9a90f54b4 100644 --- a/app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java +++ b/app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java @@ -20,6 +20,8 @@ package com.nextcloud.talk.events; +import android.support.annotation.Nullable; + import org.webrtc.MediaStream; import lombok.Data; @@ -29,7 +31,7 @@ public class MediaStreamEvent { private final MediaStream mediaStream; private final String session; - public MediaStreamEvent(MediaStream mediaStream, String session) { + public MediaStreamEvent(@Nullable MediaStream mediaStream, String session) { this.mediaStream = mediaStream; this.session = session; } diff --git a/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java b/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java new file mode 100644 index 000000000..d0d3f2fbc --- /dev/null +++ b/app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java @@ -0,0 +1,32 @@ +/* + * Nextcloud Talk application + * + * @author Mario Danic + * Copyright (C) 2017 Mario Danic + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package com.nextcloud.talk.events; + +import lombok.Data; + +@Data +public class PeerConnectionEvent { + private final String sessionId; + + public PeerConnectionEvent(String sessionId) { + this.sessionId = sessionId; + } +} diff --git a/app/src/main/java/com/nextcloud/talk/jobs/AccountRemovalJob.java b/app/src/main/java/com/nextcloud/talk/jobs/AccountRemovalJob.java index b0a9d6b3b..a487cfd47 100644 --- a/app/src/main/java/com/nextcloud/talk/jobs/AccountRemovalJob.java +++ b/app/src/main/java/com/nextcloud/talk/jobs/AccountRemovalJob.java @@ -36,6 +36,7 @@ import com.nextcloud.talk.persistence.entities.UserEntity; import com.nextcloud.talk.utils.database.user.UserUtils; import java.io.IOException; +import java.net.CookieManager; import java.util.HashMap; import javax.inject.Inject; @@ -44,6 +45,9 @@ import autodagger.AutoInjector; import io.reactivex.CompletableObserver; import io.reactivex.Observer; import io.reactivex.disposables.Disposable; +import okhttp3.JavaNetCookieJar; +import okhttp3.OkHttpClient; +import retrofit2.Retrofit; @AutoInjector(NextcloudTalkApplication.class) public class AccountRemovalJob extends Job { @@ -53,6 +57,11 @@ public class AccountRemovalJob extends Job { UserUtils userUtils; @Inject + Retrofit retrofit; + + @Inject + OkHttpClient okHttpClient; + NcApi ncApi; @NonNull @@ -68,6 +77,10 @@ public class AccountRemovalJob extends Job { pushConfigurationState = LoganSquare.parse(userEntity.getPushConfigurationState(), PushConfigurationState.class); PushConfigurationState finalPushConfigurationState = pushConfigurationState; + + ncApi = retrofit.newBuilder().client(okHttpClient.newBuilder().cookieJar(new + JavaNetCookieJar(new CookieManager())).build()).build().create(NcApi.class); + ncApi.unregisterDeviceForNotificationsWithNextcloud(ApiHelper.getCredentials(userEntity.getUsername(), userEntity.getToken()), ApiHelper.getUrlNextcloudPush(userEntity.getBaseUrl())) .subscribe(new Observer() { @@ -161,7 +174,22 @@ public class AccountRemovalJob extends Job { } catch (IOException e) { Log.d(TAG, "Something went wrong while removing job at parsing PushConfigurationState"); userUtils.deleteUser(userEntity.getUsername(), - userEntity.getBaseUrl()); + userEntity.getBaseUrl()).subscribe(new CompletableObserver() { + @Override + public void onSubscribe(Disposable d) { + + } + + @Override + public void onComplete() { + + } + + @Override + public void onError(Throwable e) { + + } + }); } } return Result.SUCCESS; diff --git a/app/src/main/java/com/nextcloud/talk/utils/PushUtils.java b/app/src/main/java/com/nextcloud/talk/utils/PushUtils.java index 17246465d..c2f9cfad0 100644 --- a/app/src/main/java/com/nextcloud/talk/utils/PushUtils.java +++ b/app/src/main/java/com/nextcloud/talk/utils/PushUtils.java @@ -41,6 +41,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.net.CookieManager; import java.security.InvalidKeyException; import java.security.Key; import java.security.KeyFactory; @@ -63,6 +64,9 @@ import javax.inject.Inject; import autodagger.AutoInjector; import io.reactivex.functions.Consumer; import io.reactivex.schedulers.Schedulers; +import okhttp3.JavaNetCookieJar; +import okhttp3.OkHttpClient; +import retrofit2.Retrofit; @AutoInjector(NextcloudTalkApplication.class) public class PushUtils { @@ -75,6 +79,11 @@ public class PushUtils { AppPreferences appPreferences; @Inject + OkHttpClient okHttpClient; + + @Inject + Retrofit retrofit; + NcApi ncApi; private File keysFile; @@ -93,7 +102,6 @@ public class PushUtils { Context.MODE_PRIVATE), "push_key.priv"); proxyServer = NextcloudTalkApplication.getSharedApplication().getResources(). getString(R.string.nc_push_server_url); - } @@ -253,6 +261,9 @@ public class PushUtils { queryMap.put("devicePublicKey", publicKey); queryMap.put("proxyServer", proxyServer); + ncApi = retrofit.newBuilder().client(okHttpClient.newBuilder().cookieJar(new + JavaNetCookieJar(new CookieManager())).build()).build().create(NcApi.class); + ncApi.registerDeviceForNotificationsWithNextcloud( ApiHelper.getCredentials(userEntity.getUsername(), userEntity.getToken()), ApiHelper.getUrlNextcloudPush(userEntity.getBaseUrl()), queryMap) diff --git a/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java index de50b2ac5..1ab336c2f 100644 --- a/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java +++ b/app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java @@ -26,6 +26,7 @@ import com.bluelinelabs.logansquare.LoganSquare; import com.nextcloud.talk.api.models.json.signaling.DataChannelMessage; import com.nextcloud.talk.api.models.json.signaling.NCIceCandidate; import com.nextcloud.talk.events.MediaStreamEvent; +import com.nextcloud.talk.events.PeerConnectionEvent; import com.nextcloud.talk.events.SessionDescriptionSendEvent; import org.greenrobot.eventbus.EventBus; @@ -50,6 +51,7 @@ public class MagicPeerConnectionWrapper { List iceCandidates = new ArrayList<>(); private List iceServers; private String sessionId; + private String localSession; private String nick; private MediaConstraints mediaConstraints; private DataChannel magicDataChannel; @@ -64,22 +66,24 @@ public class MagicPeerConnectionWrapper { String sessionId, String localSession) { this.iceServers = iceServerList; + this.localSession = localSession; peerConnection = peerConnectionFactory.createPeerConnection(iceServerList, mediaConstraints, new MagicPeerConnectionObserver()); + this.sessionId = sessionId; + this.mediaConstraints = mediaConstraints; + + magicSdpObserver = new MagicSdpObserver(); + if (sessionId.compareTo(localSession) < 0) { DataChannel.Init init = new DataChannel.Init(); init.negotiated = false; magicDataChannel = peerConnection.createDataChannel("status", init); magicDataChannel.registerObserver(new MagicDataChannelObserver()); + peerConnection.createOffer(magicSdpObserver, mediaConstraints); } - this.sessionId = sessionId; - this.mediaConstraints = mediaConstraints; - - magicSdpObserver = new MagicSdpObserver(); - } public void drainIceCandidates() { @@ -177,6 +181,9 @@ public class MagicPeerConnectionWrapper { @Override public void onSignalingChange(PeerConnection.SignalingState signalingState) { + if (signalingState.equals(PeerConnection.SignalingState.CLOSED)) { + EventBus.getDefault().post(new PeerConnectionEvent(sessionId)); + } } @Override @@ -219,6 +226,7 @@ public class MagicPeerConnectionWrapper { public void onRemoveStream(MediaStream mediaStream) { videoOn = mediaStream.videoTracks != null && mediaStream.videoTracks.size() == 1; audioOn = mediaStream.audioTracks != null && mediaStream.audioTracks.size() == 1; + EventBus.getDefault().post(new MediaStreamEvent(null, sessionId)); } @Override @@ -260,21 +268,20 @@ public class MagicPeerConnectionWrapper { @Override public void onSetSuccess() { - if (peerConnection.getRemoteDescription() == null) { - EventBus.getDefault().post(new SessionDescriptionSendEvent(peerConnection.getLocalDescription(), sessionId, - peerConnection.getLocalDescription().type.canonicalForm(), null)); + if (peerConnection != null) { + if (peerConnection.getRemoteDescription() != null) { + drainIceCandidates(); + } - } else if (peerConnection.getLocalDescription() == null && peerConnection.getRemoteDescription().type - .canonicalForm().equals - ("offer")) { - peerConnection.createAnswer(magicSdpObserver, mediaConstraints); - } else if ((peerConnection.getLocalDescription() != null && peerConnection.getRemoteDescription().type - .canonicalForm().equals - ("offer"))) { - EventBus.getDefault().post(new SessionDescriptionSendEvent(peerConnection.getLocalDescription(), sessionId, - peerConnection.getLocalDescription().type.canonicalForm(), null)); - } else if (peerConnection.getRemoteDescription() != null) { - drainIceCandidates(); + if (peerConnection.getRemoteDescription() == null) { + EventBus.getDefault().post(new SessionDescriptionSendEvent(peerConnection.getLocalDescription(), sessionId, + peerConnection.getLocalDescription().type.canonicalForm(), null)); + } else if (peerConnection.getLocalDescription() == null && sessionId.compareTo(localSession) > 0) { + peerConnection.createAnswer(magicSdpObserver, mediaConstraints); + } else if ((peerConnection.getLocalDescription() != null)) { + EventBus.getDefault().post(new SessionDescriptionSendEvent(peerConnection.getLocalDescription(), sessionId, + peerConnection.getLocalDescription().type.canonicalForm(), null)); + } } } } diff --git a/app/src/main/res/layout/activity_call.xml b/app/src/main/res/layout/activity_call.xml index f25ec4682..983302072 100644 --- a/app/src/main/res/layout/activity_call.xml +++ b/app/src/main/res/layout/activity_call.xml @@ -21,16 +21,17 @@