mirrors/nextcloud-talk-android
1
0
Fork 0
mirror of https://github.com/nextcloud/talk-android.git synced 2024-11-23 13:35:33 +03:00
Code Issues Projects Releases Packages Wiki Activity

Add screensharing support

Browse source 
Signed-off-by: Mario Danic <mario@lovelyhq.com>
This commit is contained in:
Mario Danic 2019-01-07 02:17:13 +01:00
parent b7a933e917
commit ed7cd1b788
7 changed files with 104 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

109
app/src/main/java/com/nextcloud/talk/controllers/CallController.java
View file

@ -1280,8 +1280,10 @@ public class CallController extends BaseController {
}
private void processMessage(NCSignalingMessage ncSignalingMessage) {
if (ncSignalingMessage.getRoomType().equals("video")) {
MagicPeerConnectionWrapper magicPeerConnectionWrapper = alwaysGetPeerConnectionWrapperForSessionId(ncSignalingMessage.getFrom(), false);
if (ncSignalingMessage.getRoomType().equals("video") || ncSignalingMessage.getRoomType().equals("screen")) {
MagicPeerConnectionWrapper magicPeerConnectionWrapper =
getPeerConnectionWrapperForSessionIdAndType(ncSignalingMessage.getFrom(),
ncSignalingMessage.getRoomType(),false);
String type = null;
if (ncSignalingMessage.getPayload() != null && ncSignalingMessage.getPayload().getType() != null) {
@ -1292,6 +1294,9 @@ public class CallController extends BaseController {
if (type != null) {
switch (type) {
case "unshareScreen":
endPeerConnection(ncSignalingMessage.getFrom(), true);
break;
case "offer":
case "answer":
magicPeerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick());
@ -1344,7 +1349,7 @@ public class CallController extends BaseController {
}
for (int i = 0; i < magicPeerConnectionWrapperList.size(); i++) {
endPeerConnection(magicPeerConnectionWrapperList.get(i).getSessionId());
endPeerConnection(magicPeerConnectionWrapperList.get(i).getSessionId(), false);
}
@ -1500,11 +1505,11 @@ public class CallController extends BaseController {
hasMCU = hasExternalSignalingServer && webSocketClient != null && webSocketClient.hasMCU();
for (String sessionId : newSessions) {
alwaysGetPeerConnectionWrapperForSessionId(sessionId, hasMCU && sessionId.equals(webSocketClient.getSessionId()));
getPeerConnectionWrapperForSessionIdAndType(sessionId, "video",hasMCU && sessionId.equals(webSocketClient.getSessionId()));
}
for (String sessionId : oldSesssions) {
endPeerConnection(sessionId);
endPeerConnection(sessionId, false);
}
}
@ -1545,20 +1550,35 @@ public class CallController extends BaseController {
magicPeerConnectionWrapperList.remove(magicPeerConnectionWrapper);
}
private MagicPeerConnectionWrapper alwaysGetPeerConnectionWrapperForSessionId(String sessionId, boolean publisher) {
private MagicPeerConnectionWrapper getPeerConnectionWrapperForSessionId(String sessionId, String type) {
for (int i = 0; i < magicPeerConnectionWrapperList.size(); i++) {
if (magicPeerConnectionWrapperList.get(i).getSessionId().equals(sessionId) && magicPeerConnectionWrapperList.get(i).getVideoStreamType().equals(type)) {
return magicPeerConnectionWrapperList.get(i);
}
}
return null;
}
private MagicPeerConnectionWrapper getPeerConnectionWrapperForSessionIdAndType(String sessionId, String type, boolean publisher) {
MagicPeerConnectionWrapper magicPeerConnectionWrapper;
if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) {
if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId, type)) != null) {
return magicPeerConnectionWrapper;
} else {
if (hasMCU && publisher) {
magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory,
iceServers, sdpConstraintsForMCU, sessionId, callSession, localMediaStream, true, true);
iceServers, sdpConstraintsForMCU, sessionId, callSession, localMediaStream, true, true, type);
} else if (hasMCU) {
magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory,
iceServers, sdpConstraints, sessionId, callSession, null, false, true);
iceServers, sdpConstraints, sessionId, callSession, null, false, true, type);
} else {
magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory,
iceServers, sdpConstraints, sessionId, callSession, localMediaStream, false, false);
if (!type.equals("screen")) {
magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory,
iceServers, sdpConstraints, sessionId, callSession, localMediaStream, false, false, type);
} else {
magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory,
iceServers, sdpConstraints, sessionId, callSession, null, false, false, type);
}
}
magicPeerConnectionWrapperList.add(magicPeerConnectionWrapper);
@ -1566,21 +1586,38 @@ public class CallController extends BaseController {
}
}
private MagicPeerConnectionWrapper getPeerConnectionWrapperForSessionId(String sessionId) {
private List<MagicPeerConnectionWrapper> getPeerConnectionWrapperListForSessionId(String sessionId) {
List<MagicPeerConnectionWrapper> internalList = new ArrayList<>();
for (MagicPeerConnectionWrapper magicPeerConnectionWrapper : magicPeerConnectionWrapperList) {
if (magicPeerConnectionWrapper.getSessionId().equals(sessionId)) {
return magicPeerConnectionWrapper;
internalList.add(magicPeerConnectionWrapper);
}
}
return null;
return internalList;
}
private void endPeerConnection(String sessionId) {
private void endPeerConnection(String sessionId, boolean justScreen) {
List<MagicPeerConnectionWrapper> magicPeerConnectionWrappers;
MagicPeerConnectionWrapper magicPeerConnectionWrapper;
if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null && getActivity()
if (!(magicPeerConnectionWrappers = getPeerConnectionWrapperListForSessionId(sessionId)).isEmpty() && getActivity()
!= null) {
getActivity().runOnUiThread(() -> removeMediaStream(sessionId));
deleteMagicPeerConnection(magicPeerConnectionWrapper);
for (int i = 0; i < magicPeerConnectionWrappers.size(); i++) {
magicPeerConnectionWrapper = magicPeerConnectionWrappers.get(i);
if (magicPeerConnectionWrapper.getSessionId().equals(sessionId)) {
if (magicPeerConnectionWrapper.getVideoStreamType().equals("screen")) {
MagicPeerConnectionWrapper finalMagicPeerConnectionWrapper = magicPeerConnectionWrapper;
getActivity().runOnUiThread(() -> removeMediaStream(sessionId + "+" +
finalMagicPeerConnectionWrapper.getVideoStreamType()));
deleteMagicPeerConnection(magicPeerConnectionWrapper);
} else if (!justScreen) {
MagicPeerConnectionWrapper finalMagicPeerConnectionWrapper = magicPeerConnectionWrapper;
getActivity().runOnUiThread(() -> removeMediaStream(sessionId + "+" +
finalMagicPeerConnectionWrapper.getVideoStreamType()));
deleteMagicPeerConnection(magicPeerConnectionWrapper);
}
}
}
}
}
@ -1613,7 +1650,7 @@ public class CallController extends BaseController {
public void onMessageEvent(PeerConnectionEvent peerConnectionEvent) {
if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent.PeerConnectionEventType
.PEER_CLOSED)) {
endPeerConnection(peerConnectionEvent.getSessionId());
endPeerConnection(peerConnectionEvent.getSessionId(), false);
} else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent
.PeerConnectionEventType.SENSOR_FAR) ||
peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent
@ -1630,14 +1667,14 @@ public class CallController extends BaseController {
}
} else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent
.PeerConnectionEventType.NICK_CHANGE)) {
gotNick(peerConnectionEvent.getSessionId(), peerConnectionEvent.getNick(), true);
gotNick(peerConnectionEvent.getSessionId() + "+" + peerConnectionEvent.getVideoStreamType(), peerConnectionEvent.getNick(), true);
} else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent
.PeerConnectionEventType.VIDEO_CHANGE) && !isVoiceOnlyCall) {
gotAudioOrVideoChange(true, peerConnectionEvent.getSessionId(),
gotAudioOrVideoChange(true, peerConnectionEvent.getSessionId() + "+" + peerConnectionEvent.getVideoStreamType(),
peerConnectionEvent.getChangeValue());
} else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent
.PeerConnectionEventType.AUDIO_CHANGE)) {
gotAudioOrVideoChange(false, peerConnectionEvent.getSessionId(),
gotAudioOrVideoChange(false, peerConnectionEvent.getSessionId() + "+" + peerConnectionEvent.getVideoStreamType(),
peerConnectionEvent.getChangeValue());
}
}
@ -1668,9 +1705,9 @@ public class CallController extends BaseController {
if (mediaStreamEvent.getMediaStream() != null) {
setupVideoStreamForLayout(mediaStreamEvent.getMediaStream(), mediaStreamEvent.getSession(),
mediaStreamEvent.getMediaStream().videoTracks != null
&& mediaStreamEvent.getMediaStream().videoTracks.size() > 0);
&& mediaStreamEvent.getMediaStream().videoTracks.size() > 0, mediaStreamEvent.getVideoStreamType());
} else {
setupVideoStreamForLayout(null, mediaStreamEvent.getSession(), false);
setupVideoStreamForLayout(null, mediaStreamEvent.getSession(), false, mediaStreamEvent.getVideoStreamType());
}
}
@ -1681,7 +1718,7 @@ public class CallController extends BaseController {
ncMessageWrapper.setSessionId(callSession);
NCSignalingMessage ncSignalingMessage = new NCSignalingMessage();
ncSignalingMessage.setTo(sessionDescriptionSend.getPeerId());
ncSignalingMessage.setRoomType("video");
ncSignalingMessage.setRoomType(sessionDescriptionSend.getVideoStreamType());
ncSignalingMessage.setType(sessionDescriptionSend.getType());
NCMessagePayload ncMessagePayload = new NCMessagePayload();
ncMessagePayload.setType(sessionDescriptionSend.getType());
@ -1768,7 +1805,7 @@ public class CallController extends BaseController {
private void setupAvatarForSession(String session) {
if (remoteRenderersLayout != null) {
RelativeLayout relativeLayout = remoteRenderersLayout.findViewWithTag(session);
RelativeLayout relativeLayout = remoteRenderersLayout.findViewWithTag(session + "+video");
if (relativeLayout != null) {
ImageView avatarImageView = relativeLayout.findViewById(R.id.avatarImageView);
@ -1791,14 +1828,14 @@ public class CallController extends BaseController {
}
}
private void setupVideoStreamForLayout(@Nullable MediaStream mediaStream, String session, boolean enable) {
private void setupVideoStreamForLayout(@Nullable MediaStream mediaStream, String session, boolean enable, String videoStreamType) {
boolean isInitialLayoutSetupForPeer = false;
if (remoteRenderersLayout.findViewWithTag(session) == null) {
setupNewPeerLayout(session);
setupNewPeerLayout(session, videoStreamType);
isInitialLayoutSetupForPeer = true;
}
RelativeLayout relativeLayout = remoteRenderersLayout.findViewWithTag(session);
RelativeLayout relativeLayout = remoteRenderersLayout.findViewWithTag(session + "+" + videoStreamType);
SurfaceViewRenderer surfaceViewRenderer = relativeLayout.findViewById(R.id.surface_view);
ImageView imageView = relativeLayout.findViewById(R.id.avatarImageView);
@ -1850,13 +1887,13 @@ public class CallController extends BaseController {
}
}
private void setupNewPeerLayout(String session) {
if (remoteRenderersLayout.findViewWithTag(session) == null && getActivity() != null) {
private void setupNewPeerLayout(String session, String type) {
if (remoteRenderersLayout.findViewWithTag(session + "+" + type) == null && getActivity() != null) {
getActivity().runOnUiThread(() -> {
RelativeLayout relativeLayout = (RelativeLayout)
getActivity().getLayoutInflater().inflate(R.layout.call_item, remoteRenderersLayout,
false);
relativeLayout.setTag(session);
relativeLayout.setTag(session + "+" + type);
SurfaceViewRenderer surfaceViewRenderer = relativeLayout.findViewById(R.id
.surface_view);
@ -1869,12 +1906,14 @@ public class CallController extends BaseController {
surfaceViewRenderer.setOnClickListener(videoOnClickListener);
remoteRenderersLayout.addView(relativeLayout);
if (hasExternalSignalingServer) {
gotNick(session, webSocketClient.getDisplayNameForSession(session), false);
gotNick(session + "+" + type, webSocketClient.getDisplayNameForSession(session), false);
} else {
gotNick(session, getPeerConnectionWrapperForSessionId(session).getNick(), false);
gotNick(session + "+" + type, getPeerConnectionWrapperForSessionIdAndType(session, "video", false).getNick(), false);
}
setupAvatarForSession(session);
if (type.equals("video")) {
setupAvatarForSession(session);
}
callControls.setZ(100.0f);
});

4
app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java
View file

@ -29,9 +29,11 @@ import lombok.Data;
public class MediaStreamEvent {
private final MediaStream mediaStream;
private final String session;
private final String videoStreamType;
public MediaStreamEvent(@Nullable MediaStream mediaStream, String session) {
public MediaStreamEvent(@Nullable MediaStream mediaStream, String session, String videoStreamType) {
this.mediaStream = mediaStream;
this.session = session;
this.videoStreamType = videoStreamType;
}
}

4
app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java
View file

@ -29,13 +29,15 @@ public class PeerConnectionEvent {
private final String sessionId;
private final String nick;
private final Boolean changeValue;
private final String videoStreamType;
public PeerConnectionEvent(PeerConnectionEventType peerConnectionEventType, @Nullable String sessionId,
@Nullable String nick, Boolean changeValue) {
@Nullable String nick, Boolean changeValue, @Nullable String videoStreamType) {
this.peerConnectionEventType = peerConnectionEventType;
this.nick = nick;
this.changeValue = changeValue;
this.sessionId = sessionId;
this.videoStreamType = videoStreamType;
}
public enum PeerConnectionEventType {

4
app/src/main/java/com/nextcloud/talk/events/SessionDescriptionSendEvent.java
View file

@ -35,12 +35,14 @@ public class SessionDescriptionSendEvent {
private final String type;
@Nullable
private final NCIceCandidate ncIceCandidate;
private final String videoStreamType;
public SessionDescriptionSendEvent(@Nullable SessionDescription sessionDescription, String peerId, String type,
@Nullable NCIceCandidate ncIceCandidate) {
@Nullable NCIceCandidate ncIceCandidate, @Nullable String videoStreamType) {
this.sessionDescription = sessionDescription;
this.peerId = peerId;
this.type = type;
this.ncIceCandidate = ncIceCandidate;
this.videoStreamType = videoStreamType;
}
}

4
app/src/main/java/com/nextcloud/talk/webrtc/MagicAudioManager.java
View file

@ -176,7 +176,7 @@ public class MagicAudioManager {
setAudioDeviceInternal(MagicAudioManager.AudioDevice.EARPIECE);
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
.SENSOR_NEAR, null, null, null));
.SENSOR_NEAR, null, null, null, null));
} else {
// Sensor reports that a "handset is removed from a person's ear", or
@ -184,7 +184,7 @@ public class MagicAudioManager {
setAudioDeviceInternal(MagicAudioManager.AudioDevice.SPEAKER_PHONE);
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
.SENSOR_FAR, null, null, null));
.SENSOR_FAR, null, null, null, null));
}
}
}

30
app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java
View file

@ -71,14 +71,16 @@ public class MagicPeerConnectionWrapper {
private MediaStream localMediaStream;
private boolean isMCUPublisher;
private String videoStreamType;
public MagicPeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory,
List<PeerConnection.IceServer> iceServerList,
MediaConstraints mediaConstraints,
String sessionId, String localSession, @Nullable MediaStream mediaStream,
boolean isMCUPublisher, boolean hasMCU) {
boolean isMCUPublisher, boolean hasMCU, String videoStreamType) {
this.localMediaStream = mediaStream;
this.videoStreamType = videoStreamType;
this.sessionId = sessionId;
this.mediaConstraints = mediaConstraints;
@ -114,6 +116,10 @@ public class MagicPeerConnectionWrapper {
}
}
public String getVideoStreamType() {
return videoStreamType;
}
public void removePeerConnection() {
if (magicDataChannel != null) {
magicDataChannel.dispose();
@ -243,32 +249,32 @@ public class MagicPeerConnectionWrapper {
if (!internalNick.equals(nick)) {
setNick(nick);
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
.NICK_CHANGE, sessionId, getNick(), null));
.NICK_CHANGE, sessionId, getNick(), null, videoStreamType));
}
} else {
if (dataChannelMessage.getPayload() != null) {
HashMap<String, String> payloadHashMap = (HashMap<String, String>) dataChannelMessage.getPayload();
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
.NICK_CHANGE, payloadHashMap.get("userid"), payloadHashMap.get("name"), null));
.NICK_CHANGE, payloadHashMap.get("userid"), payloadHashMap.get("name"), null, videoStreamType));
}
}
} else if ("audioOn".equals(dataChannelMessage.getType())) {
remoteAudioOn = true;
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
.AUDIO_CHANGE, sessionId, null, remoteAudioOn));
.AUDIO_CHANGE, sessionId, null, remoteAudioOn, videoStreamType));
} else if ("audioOff".equals(dataChannelMessage.getType())) {
remoteAudioOn = false;
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
.AUDIO_CHANGE, sessionId, null, remoteAudioOn));
.AUDIO_CHANGE, sessionId, null, remoteAudioOn, videoStreamType));
} else if ("videoOn".equals(dataChannelMessage.getType())) {
remoteVideoOn = true;
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
.VIDEO_CHANGE, sessionId, null, remoteVideoOn));
.VIDEO_CHANGE, sessionId, null, remoteVideoOn, videoStreamType));
} else if ("videoOff".equals(dataChannelMessage.getType())) {
remoteVideoOn = false;
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
.VIDEO_CHANGE, sessionId, null, remoteVideoOn));
.VIDEO_CHANGE, sessionId, null, remoteVideoOn, videoStreamType));
}
} catch (IOException e) {
Log.d(TAG, "Failed to parse data channel message");
@ -290,7 +296,7 @@ public class MagicPeerConnectionWrapper {
.PEER_CONNECTED, sessionId, null, null));*/
if (!isMCUPublisher) {
EventBus.getDefault().post(new MediaStreamEvent(remoteMediaStream, sessionId));
EventBus.getDefault().post(new MediaStreamEvent(remoteMediaStream, sessionId, videoStreamType));
}
if (hasInitiated) {
@ -299,7 +305,7 @@ public class MagicPeerConnectionWrapper {
} else if (iceConnectionState.equals(PeerConnection.IceConnectionState.CLOSED)) {
EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
.PEER_CLOSED, sessionId, null, null));
.PEER_CLOSED, sessionId, null, null, videoStreamType));
}
}
@ -320,7 +326,7 @@ public class MagicPeerConnectionWrapper {
ncIceCandidate.setSdpMLineIndex(iceCandidate.sdpMLineIndex);
ncIceCandidate.setCandidate(iceCandidate.sdp);
EventBus.getDefault().post(new SessionDescriptionSendEvent(null, sessionId,
"candidate", ncIceCandidate));
"candidate", ncIceCandidate, videoStreamType));
}
@Override
@ -336,7 +342,7 @@ public class MagicPeerConnectionWrapper {
@Override
public void onRemoveStream(MediaStream mediaStream) {
if (!isMCUPublisher) {
EventBus.getDefault().post(new MediaStreamEvent(null, sessionId));
EventBus.getDefault().post(new MediaStreamEvent(null, sessionId, videoStreamType));
}
}
@ -383,7 +389,7 @@ public class MagicPeerConnectionWrapper {
EventBus.getDefault().post(new SessionDescriptionSendEvent(sessionDescriptionWithPreferredCodec, sessionId,
sessionDescription.type.canonicalForm().toLowerCase(), null));
sessionDescription.type.canonicalForm().toLowerCase(), null, videoStreamType));
if (peerConnection != null) {
peerConnection.setLocalDescription(magicSdpObserver, sessionDescriptionWithPreferredCodec);

2
app/src/main/java/com/nextcloud/talk/webrtc/MagicWebRTCUtils.java
View file

@ -129,7 +129,7 @@ public class MagicWebRTCUtils {
}
Log.d(TAG, "Change media description from: " + lines[mLineIndex] + " to " + newMLine);
lines[mLineIndex] = newMLine;
return joinString(Arrays.asList(lines), "\r\n", true /* delimiterAtEnd */);
return joinString(Arrays.asList(lines), "\r\n", true);
}
/**