diff --git a/changelog.d/5911.feature b/changelog.d/5911.feature
new file mode 100644
index 0000000000..368a3b4056
--- /dev/null
+++ b/changelog.d/5911.feature
@@ -0,0 +1 @@
+Screen sharing over WebRTC
diff --git a/vector/src/main/java/im/vector/app/features/VectorFeatures.kt b/vector/src/main/java/im/vector/app/features/VectorFeatures.kt
index 9d54475e8c..42693a53f9 100644
--- a/vector/src/main/java/im/vector/app/features/VectorFeatures.kt
+++ b/vector/src/main/java/im/vector/app/features/VectorFeatures.kt
@@ -44,5 +44,5 @@ class DefaultVectorFeatures : VectorFeatures {
override fun isOnboardingPersonalizeEnabled() = false
override fun isOnboardingCombinedRegisterEnabled() = false
override fun isLiveLocationEnabled(): Boolean = false
- override fun isScreenSharingEnabled(): Boolean = false
+ override fun isScreenSharingEnabled(): Boolean = true
}
diff --git a/vector/src/main/java/im/vector/app/features/call/CallControlsView.kt b/vector/src/main/java/im/vector/app/features/call/CallControlsView.kt
index b3fc36e5bc..f0158fc4d6 100644
--- a/vector/src/main/java/im/vector/app/features/call/CallControlsView.kt
+++ b/vector/src/main/java/im/vector/app/features/call/CallControlsView.kt
@@ -89,6 +89,8 @@ class CallControlsView @JvmOverloads constructor(
views.videoToggleIcon.setImageResource(R.drawable.ic_video_off)
views.videoToggleIcon.contentDescription = resources.getString(R.string.a11y_start_camera)
}
+ views.videoToggleIcon.isEnabled = !state.isSharingScreen
+ views.videoToggleIcon.alpha = if (state.isSharingScreen) 0.5f else 1f
when (callState) {
is CallState.LocalRinging -> {
diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt
index ea9adcde85..a904658e9c 100644
--- a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt
+++ b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt
@@ -24,6 +24,7 @@ import android.content.Intent
import android.content.Intent.FLAG_ACTIVITY_CLEAR_TOP
import android.content.res.Configuration
import android.graphics.Color
+import android.media.projection.MediaProjection
import android.media.projection.MediaProjectionManager
import android.os.Build
import android.os.Bundle
@@ -32,6 +33,7 @@ import android.util.Rational
import android.view.MenuItem
import android.view.View
import android.view.WindowManager
+import androidx.activity.result.ActivityResult
import androidx.annotation.StringRes
import androidx.core.content.ContextCompat
import androidx.core.content.getSystemService
@@ -76,6 +78,7 @@ import org.matrix.android.sdk.api.session.call.TurnServerResponse
import org.matrix.android.sdk.api.session.room.model.call.EndCallReason
import org.webrtc.EglBase
import org.webrtc.RendererCommon
+import org.webrtc.ScreenCapturerAndroid
import timber.log.Timber
import javax.inject.Inject
@@ -161,6 +164,9 @@ class VectorCallActivity : VectorBaseActivity<ActivityCallBinding>(), CallContro
}
}
}
+
+ // Bind to service in case of user killed the app while there is an ongoing call
+ bindToScreenCaptureService()
}
override fun onNewIntent(intent: Intent?) {
@@ -636,18 +642,40 @@ class VectorCallActivity : VectorBaseActivity<ActivityCallBinding>(), CallContro
private val screenSharingPermissionActivityResultLauncher = registerStartForActivityResult { activityResult ->
if (activityResult.resultCode == Activity.RESULT_OK) {
- callViewModel.handle(VectorCallViewActions.StartScreenSharing)
- // We need to start a foreground service with a sticky notification during screen sharing
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
- ContextCompat.startForegroundService(
- this,
- Intent(this, ScreenCaptureService::class.java)
- )
- screenCaptureServiceConnection.bind()
+ // We need to start a foreground service with a sticky notification during screen sharing
+ startScreenSharingService(activityResult)
+ } else {
+ startScreenSharing(activityResult)
}
}
}
+ private fun startScreenSharing(activityResult: ActivityResult) {
+ val videoCapturer = ScreenCapturerAndroid(activityResult.data, object : MediaProjection.Callback() {
+ override fun onStop() {
+ Timber.i("User revoked the screen capturing permission")
+ }
+ })
+ callViewModel.handle(VectorCallViewActions.StartScreenSharing(videoCapturer))
+ }
+
+ private fun startScreenSharingService(activityResult: ActivityResult) {
+ ContextCompat.startForegroundService(
+ this,
+ Intent(this, ScreenCaptureService::class.java)
+ )
+ bindToScreenCaptureService(activityResult)
+ }
+
+ private fun bindToScreenCaptureService(activityResult: ActivityResult? = null) {
+ screenCaptureServiceConnection.bind(object : ScreenCaptureServiceConnection.Callback {
+ override fun onServiceConnected() {
+ activityResult?.let { startScreenSharing(it) }
+ }
+ })
+ }
+
private fun handleShowScreenSharingPermissionDialog() {
getSystemService<MediaProjectionManager>()?.let {
navigator.openScreenSharingPermissionDialog(it.createScreenCaptureIntent(), screenSharingPermissionActivityResultLauncher)
diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt
index c84f733b9a..cec118f296 100644
--- a/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt
+++ b/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt
@@ -19,6 +19,7 @@ package im.vector.app.features.call
import im.vector.app.core.platform.VectorViewModelAction
import im.vector.app.features.call.audio.CallAudioManager
import im.vector.app.features.call.transfer.CallTransferResult
+import org.webrtc.VideoCapturer
sealed class VectorCallViewActions : VectorViewModelAction {
object EndCall : VectorCallViewActions()
@@ -41,5 +42,5 @@ sealed class VectorCallViewActions : VectorViewModelAction {
data class CallTransferSelectionResult(val callTransferResult: CallTransferResult) : VectorCallViewActions()
object TransferCall : VectorCallViewActions()
object ToggleScreenSharing : VectorCallViewActions()
- object StartScreenSharing : VectorCallViewActions()
+ data class StartScreenSharing(val videoCapturer: VideoCapturer) : VectorCallViewActions()
}
diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt
index 55a0219bfe..e2cedbe1b0 100644
--- a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt
+++ b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt
@@ -145,9 +145,10 @@ class VectorCallViewModel @AssistedInject constructor(
override fun onCallEnded(callId: String) {
withState { state ->
if (state.otherKnownCallInfo?.callId == callId) {
- setState { copy(otherKnownCallInfo = null) }
+ setState { copy(otherKnownCallInfo = null, isSharingScreen = false) }
}
}
+ _viewEvents.post(VectorCallViewEvents.StopScreenSharingService)
}
override fun onCurrentCallChange(call: WebRtcCall?) {
@@ -156,9 +157,10 @@ class VectorCallViewModel @AssistedInject constructor(
}
}
- override fun onAudioDevicesChange() {
- val currentSoundDevice = callManager.audioManager.selectedDevice ?: return
- if (currentSoundDevice == CallAudioManager.Device.Phone) {
+ override fun onAudioDevicesChange() = withState { state ->
+ val currentSoundDevice = callManager.audioManager.selectedDevice ?: return@withState
+ val webRtcCall = callManager.getCallById(state.callId)
+ if (webRtcCall != null && shouldActivateProximitySensor(webRtcCall)) {
proximityManager.start()
} else {
proximityManager.stop()
@@ -205,7 +207,7 @@ class VectorCallViewModel @AssistedInject constructor(
callManager.addListener(callManagerListener)
webRtcCall.addListener(callListener)
val currentSoundDevice = callManager.audioManager.selectedDevice
- if (currentSoundDevice == CallAudioManager.Device.Phone) {
+ if (shouldActivateProximitySensor(webRtcCall)) {
proximityManager.start()
}
setState {
@@ -224,13 +226,18 @@ class VectorCallViewModel @AssistedInject constructor(
formattedDuration = webRtcCall.formattedDuration(),
isHD = webRtcCall.mxCall.isVideoCall && webRtcCall.currentCaptureFormat() is CaptureFormat.HD,
canOpponentBeTransferred = webRtcCall.mxCall.capabilities.supportCallTransfer(),
- transferee = computeTransfereeState(webRtcCall.mxCall)
+ transferee = computeTransfereeState(webRtcCall.mxCall),
+ isSharingScreen = webRtcCall.isSharingScreen()
)
}
updateOtherKnownCall(webRtcCall)
}
}
+ private fun shouldActivateProximitySensor(webRtcCall: WebRtcCall): Boolean {
+ return callManager.audioManager.selectedDevice == CallAudioManager.Device.Phone && !webRtcCall.isSharingScreen()
+ }
+
private fun WebRtcCall.extractCallInfo(): VectorCallViewState.CallInfo {
val assertedIdentity = this.remoteAssertedIdentity
val matrixItem = if (assertedIdentity != null) {
@@ -349,7 +356,8 @@ class VectorCallViewModel @AssistedInject constructor(
handleToggleScreenSharing(state.isSharingScreen)
}
is VectorCallViewActions.StartScreenSharing -> {
- call?.startSharingScreen()
+ call?.startSharingScreen(action.videoCapturer)
+ proximityManager.stop()
setState {
copy(isSharingScreen = true)
}
@@ -366,6 +374,9 @@ class VectorCallViewModel @AssistedInject constructor(
_viewEvents.post(
VectorCallViewEvents.StopScreenSharingService
)
+ if (callManager.audioManager.selectedDevice == CallAudioManager.Device.Phone) {
+ proximityManager.start()
+ }
} else {
_viewEvents.post(
VectorCallViewEvents.ShowScreenSharingPermissionDialog
diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/ScreenCaptureServiceConnection.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/ScreenCaptureServiceConnection.kt
index 922e9676a8..aa7c7f450a 100644
--- a/vector/src/main/java/im/vector/app/features/call/webrtc/ScreenCaptureServiceConnection.kt
+++ b/vector/src/main/java/im/vector/app/features/call/webrtc/ScreenCaptureServiceConnection.kt
@@ -27,11 +27,20 @@ class ScreenCaptureServiceConnection @Inject constructor(
private val context: Context
) : ServiceConnection {
+ interface Callback {
+ fun onServiceConnected()
+ }
+
private var isBound = false
private var screenCaptureService: ScreenCaptureService? = null
+ private var callback: Callback? = null
- fun bind() {
- if (!isBound) {
+ fun bind(callback: Callback) {
+ this.callback = callback
+
+ if (isBound) {
+ callback.onServiceConnected()
+ } else {
Intent(context, ScreenCaptureService::class.java).also { intent ->
context.bindService(intent, this, 0)
}
@@ -45,10 +54,12 @@ class ScreenCaptureServiceConnection @Inject constructor(
override fun onServiceConnected(className: ComponentName, binder: IBinder) {
screenCaptureService = (binder as ScreenCaptureService.LocalBinder).getService()
isBound = true
+ callback?.onServiceConnected()
}
override fun onServiceDisconnected(className: ComponentName) {
isBound = false
screenCaptureService = null
+ callback = null
}
}
diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt
index f0db3e199f..5a100edcf2 100644
--- a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt
+++ b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt
@@ -80,10 +80,12 @@ import org.webrtc.MediaConstraints
import org.webrtc.MediaStream
import org.webrtc.PeerConnection
import org.webrtc.PeerConnectionFactory
+import org.webrtc.RtpSender
import org.webrtc.RtpTransceiver
import org.webrtc.SessionDescription
import org.webrtc.SurfaceTextureHelper
import org.webrtc.SurfaceViewRenderer
+import org.webrtc.VideoCapturer
import org.webrtc.VideoSource
import org.webrtc.VideoTrack
import timber.log.Timber
@@ -95,6 +97,7 @@ import kotlin.coroutines.CoroutineContext
private const val STREAM_ID = "userMedia"
private const val AUDIO_TRACK_ID = "${STREAM_ID}a0"
private const val VIDEO_TRACK_ID = "${STREAM_ID}v0"
+private const val SCREEN_TRACK_ID = "${STREAM_ID}s0"
private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints()
private const val INVITE_TIMEOUT_IN_MS = 60_000L
@@ -153,13 +156,16 @@ class WebRtcCall(
private var makingOffer: Boolean = false
private var ignoreOffer: Boolean = false
- private var videoCapturer: CameraVideoCapturer? = null
+ private var videoCapturer: VideoCapturer? = null
private val availableCamera = ArrayList<CameraProxy>()
private var cameraInUse: CameraProxy? = null
private var currentCaptureFormat: CaptureFormat = CaptureFormat.HD
private var cameraAvailabilityCallback: CameraManager.AvailabilityCallback? = null
+ private var videoSender: RtpSender? = null
+ private var screenSender: RtpSender? = null
+
private val timer = CountUpTimer(1000L).apply {
tickListener = object : CountUpTimer.TickListener {
override fun onTick(milliseconds: Long) {
@@ -617,7 +623,7 @@ class WebRtcCall(
val videoTrack = peerConnectionFactory.createVideoTrack(VIDEO_TRACK_ID, videoSource)
Timber.tag(loggerTag.value).v("Add video track $VIDEO_TRACK_ID to call ${mxCall.callId}")
videoTrack.setEnabled(true)
- peerConnection?.addTrack(videoTrack, listOf(STREAM_ID))
+ videoSender = peerConnection?.addTrack(videoTrack, listOf(STREAM_ID))
localVideoSource = videoSource
localVideoTrack = videoTrack
}
@@ -718,7 +724,7 @@ class WebRtcCall(
Timber.tag(loggerTag.value).v("switchCamera")
if (mxCall.state is CallState.Connected && mxCall.isVideoCall) {
val oppositeCamera = getOppositeCameraIfAny() ?: return@launch
- videoCapturer?.switchCamera(
+ (videoCapturer as? CameraVideoCapturer)?.switchCamera(
object : CameraVideoCapturer.CameraSwitchHandler {
// Invoked on success. |isFrontCamera| is true if the new camera is front facing.
override fun onCameraSwitchDone(isFrontCamera: Boolean) {
@@ -766,12 +772,60 @@ class WebRtcCall(
return currentCaptureFormat
}
- fun startSharingScreen() {
- // TODO. Will be handled within the next PR.
+ fun startSharingScreen(videoCapturer: VideoCapturer) {
+ val factory = peerConnectionFactoryProvider.get() ?: return
+
+ this.videoCapturer = videoCapturer
+
+ val localMediaStream = factory.createLocalMediaStream(STREAM_ID)
+ val videoSource = factory.createVideoSource(videoCapturer.isScreencast)
+
+ startCapturingScreen(videoCapturer, videoSource)
+
+ removeLocalSurfaceRenderers()
+
+ showScreenLocally(factory, videoSource, localMediaStream)
+
+ videoSender?.let { removeStream(it) }
+
+ screenSender = peerConnection?.addTrack(localVideoTrack, listOf(STREAM_ID))
}
fun stopSharingScreen() {
- // TODO. Will be handled within the next PR.
+ localVideoTrack?.setEnabled(false)
+ screenSender?.let { removeStream(it) }
+ if (mxCall.isVideoCall) {
+ peerConnectionFactoryProvider.get()?.let { configureVideoTrack(it) }
+ }
+ updateMuteStatus()
+ sessionScope?.launch(dispatcher) { attachViewRenderersInternal() }
+ }
+
+ private fun removeStream(sender: RtpSender) {
+ peerConnection?.removeTrack(sender)
+ }
+
+ private fun showScreenLocally(factory: PeerConnectionFactory, videoSource: VideoSource?, localMediaStream: MediaStream?) {
+ localVideoTrack = factory.createVideoTrack(SCREEN_TRACK_ID, videoSource).apply { setEnabled(true) }
+ localMediaStream?.addTrack(localVideoTrack)
+ localSurfaceRenderers.forEach { it.get()?.let { localVideoTrack?.addSink(it) } }
+ }
+
+ private fun removeLocalSurfaceRenderers() {
+ localSurfaceRenderers.forEach { it.get()?.let { localVideoTrack?.removeSink(it) } }
+ }
+
+ private fun startCapturingScreen(videoCapturer: VideoCapturer, videoSource: VideoSource) {
+ val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext)
+ videoCapturer.initialize(surfaceTextureHelper, context, videoSource.capturerObserver)
+ videoCapturer.startCapture(currentCaptureFormat.width, currentCaptureFormat.height, currentCaptureFormat.fps)
+ }
+
+ /**
+ * Returns true if the user is sharing the screen, false otherwise.
+ */
+ fun isSharingScreen(): Boolean {
+ return localVideoTrack?.enabled().orFalse() && localVideoTrack?.id() == SCREEN_TRACK_ID
}
private suspend fun release() {