From dc19652c2b10cfcbf3d71fdec42de0a176f19f5a Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 5 May 2020 11:46:21 +0200 Subject: [PATCH] WIP refact WIP TMP WIP --- matrix-sdk-android/build.gradle | 4 + .../matrix/android/api/session/Session.kt | 6 + .../android/api/session/call/CallService.kt | 57 + .../android/api/session/call/CallsListener.kt | 47 + .../android/api/session/call/EglUtils.kt | 55 + .../api/session/call/PeerSignalingClient.kt | 66 + .../session/call/RoomConnectionParameter.kt | 40 + .../android/api/session/call/TurnServer.kt | 28 + .../android/api/session/call/VoipApi.kt | 28 + .../api/session/events/model/EventType.kt | 1 - .../room/model/call/CallInviteContent.kt | 35 +- .../android/internal/crypto/CryptoModule.kt | 5 + .../internal/crypto/tasks/SendEventTask.kt | 79 + .../internal/session/DefaultSession.kt | 6 +- .../android/internal/session/SessionModule.kt | 16 +- .../session/call/CallEventObserver.kt | 67 + .../session/call/CallEventsObserverTask.kt | 88 ++ .../session/call/DefaultCallService.kt | 206 +++ .../internal/session/room/RoomModule.kt | 6 + .../session/room/send/DefaultSendService.kt | 21 +- .../session/room/send/RoomEventSender.kt | 72 + vector/build.gradle | 3 + .../riotx/features/debug/DebugMenuActivity.kt | 4 +- vector/src/main/AndroidManifest.xml | 16 + .../java/im/vector/riotx/VectorApplication.kt | 4 + .../vector/riotx/core/di/ScreenComponent.kt | 2 + .../vector/riotx/core/di/VectorComponent.kt | 3 + .../riotx/features/call/CallConnection.kt | 42 + .../riotx/features/call/CallFragment.java | 133 ++ .../features/call/PeerConnectionClient.java | 1374 +++++++++++++++++ .../call/PeerConnectionObserverAdapter.kt | 70 + .../riotx/features/call/SdpObserverAdapter.kt | 39 + .../riotx/features/call/VectorCallActivity.kt | 434 ++++++ .../features/call/VectorCallViewModel.kt | 124 ++ .../features/call/VectorConnectionService.kt | 52 + .../call/WebRtcPeerConnectionManager.kt | 372 +++++ .../home/room/detail/RoomDetailAction.kt | 1 + .../home/room/detail/RoomDetailFragment.kt | 12 + .../home/room/detail/RoomDetailViewModel.kt | 2 + .../timeline/format/NoticeEventFormatter.kt | 2 +- vector/src/main/res/drawable/ic_phone.xml | 14 + vector/src/main/res/layout/activity_call.xml | 33 + vector/src/main/res/layout/fragment_call.xml | 63 + vector/src/main/res/menu/menu_timeline.xml | 8 + 44 files changed, 3710 insertions(+), 30 deletions(-) create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/EglUtils.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/RoomConnectionParameter.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServer.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/tasks/SendEventTask.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt create mode 100644 matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/RoomEventSender.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/CallConnection.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/CallFragment.java create mode 100644 vector/src/main/java/im/vector/riotx/features/call/PeerConnectionClient.java create mode 100644 vector/src/main/java/im/vector/riotx/features/call/PeerConnectionObserverAdapter.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/SdpObserverAdapter.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/VectorConnectionService.kt create mode 100644 vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt create mode 100644 vector/src/main/res/drawable/ic_phone.xml create mode 100644 vector/src/main/res/layout/activity_call.xml create mode 100644 vector/src/main/res/layout/fragment_call.xml diff --git a/matrix-sdk-android/build.gradle b/matrix-sdk-android/build.gradle index e4251a733c..b44cb595d0 100644 --- a/matrix-sdk-android/build.gradle +++ b/matrix-sdk-android/build.gradle @@ -162,6 +162,10 @@ dependencies { // Phone number https://github.com/google/libphonenumber implementation 'com.googlecode.libphonenumber:libphonenumber:8.10.23' + // Web RTC + // TODO meant for development purposes only + implementation 'org.webrtc:google-webrtc:1.0.+' + debugImplementation 'com.airbnb.okreplay:okreplay:1.5.0' releaseImplementation 'com.airbnb.okreplay:noop:1.5.0' androidTestImplementation 'com.airbnb.okreplay:espresso:1.5.0' diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt index 5736e78a30..ab1588f8c5 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/Session.kt @@ -24,6 +24,7 @@ import im.vector.matrix.android.api.pushrules.PushRuleService import im.vector.matrix.android.api.session.account.AccountService import im.vector.matrix.android.api.session.accountdata.AccountDataService import im.vector.matrix.android.api.session.cache.CacheService +import im.vector.matrix.android.api.session.call.CallService import im.vector.matrix.android.api.session.content.ContentUploadStateTracker import im.vector.matrix.android.api.session.content.ContentUrlResolver import im.vector.matrix.android.api.session.crypto.CryptoService @@ -165,6 +166,11 @@ interface Session : */ fun integrationManagerService(): IntegrationManagerService + /** + * Returns the cryptoService associated with the session + */ + fun callService(): CallService + /** * Add a listener to the session. * @param listener the listener to add. diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt new file mode 100644 index 0000000000..5e3f331148 --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallService.kt @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.api.session.call + +import im.vector.matrix.android.api.MatrixCallback +import org.webrtc.IceCandidate +import org.webrtc.SessionDescription + +interface CallService { + + + fun getTurnServer(callback: MatrixCallback) + + fun isCallSupportedInRoom(roomId: String) : Boolean + + + /** + * Send offer SDP to the other participant. + */ + fun sendOfferSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) + + /** + * Send answer SDP to the other participant. + */ + fun sendAnswerSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) + + /** + * Send Ice candidate to the other participant. + */ + fun sendLocalIceCandidates(callId: String, roomId: String, candidates: List) + + /** + * Send removed ICE candidates to the other participant. + */ + fun sendLocalIceCandidateRemovals(callId: String, roomId: String, candidates: List) + + + fun addCallListener(listener: CallsListener) + + fun removeCallListener(listener: CallsListener) + + +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt new file mode 100644 index 0000000000..556555369a --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/CallsListener.kt @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.api.session.call + +import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent +import im.vector.matrix.android.api.session.room.model.call.CallInviteContent + +interface CallsListener { +// /** +// * Called when there is an incoming call within the room. +// * @param peerSignalingClient the incoming call +// */ +// fun onIncomingCall(peerSignalingClient: PeerSignalingClient) +// +// /** +// * An outgoing call is started. +// * +// * @param peerSignalingClient the outgoing call +// */ +// fun onOutgoingCall(peerSignalingClient: PeerSignalingClient) +// +// /** +// * Called when a called has been hung up +// * +// * @param peerSignalingClient the incoming call +// */ +// fun onCallHangUp(peerSignalingClient: PeerSignalingClient) + + fun onCallInviteReceived(signalingRoomId: String, callInviteContent: CallInviteContent) + + fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) + +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/EglUtils.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/EglUtils.kt new file mode 100644 index 0000000000..bd1d95ae6f --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/EglUtils.kt @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package im.vector.matrix.android.api.session.call + +import org.webrtc.EglBase +import timber.log.Timber + +/** + * The root [EglBase] instance shared by the entire application for + * the sake of reducing the utilization of system resources (such as EGL + * contexts) + * by performing a runtime check. + */ +object EglUtils { + + // TODO how do we release that? + + /** + * Lazily creates and returns the one and only [EglBase] which will + * serve as the root for all contexts that are needed. + */ + @get:Synchronized var rootEglBase: EglBase? = null + get() { + if (field == null) { + val configAttributes = EglBase.CONFIG_PLAIN + try { + field = EglBase.createEgl14(configAttributes) + ?: EglBase.createEgl10(configAttributes) // Fall back to EglBase10. + } catch (ex: Throwable) { + Timber.e(ex, "Failed to create EglBase") + } + } + return field + } + private set + + val rootEglBaseContext: EglBase.Context? + get() { + val eglBase = rootEglBase + return eglBase?.eglBaseContext + } +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt new file mode 100644 index 0000000000..9a948adbb8 --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/PeerSignalingClient.kt @@ -0,0 +1,66 @@ +///* +// * Copyright (c) 2020 New Vector Ltd +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//package im.vector.matrix.android.api.session.call +// +//import im.vector.matrix.android.api.MatrixCallback +//import org.webrtc.IceCandidate +//import org.webrtc.SessionDescription +// +//interface PeerSignalingClient { +// +// val callID: String +// +// fun addListener(listener: SignalingListener) +// +// /** +// * Send offer SDP to the other participant. +// */ +// fun sendOfferSdp(sdp: SessionDescription, callback: MatrixCallback) +// +// /** +// * Send answer SDP to the other participant. +// */ +// fun sendAnswerSdp(sdp: SessionDescription, callback: MatrixCallback) +// +// /** +// * Send Ice candidate to the other participant. +// */ +// fun sendLocalIceCandidates(candidates: List) +// +// /** +// * Send removed ICE candidates to the other participant. +// */ +// fun sendLocalIceCandidateRemovals(candidates: List) +// +// +// interface SignalingListener { +// /** +// * Callback fired once remote SDP is received. +// */ +// fun onRemoteDescription(sdp: SessionDescription) +// +// /** +// * Callback fired once remote Ice candidate is received. +// */ +// fun onRemoteIceCandidate(candidate: IceCandidate) +// +// /** +// * Callback fired once remote Ice candidate removals are received. +// */ +// fun onRemoteIceCandidatesRemoved(candidates: List) +// } +//} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/RoomConnectionParameter.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/RoomConnectionParameter.kt new file mode 100644 index 0000000000..09769efc55 --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/RoomConnectionParameter.kt @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.api.session.call + +import org.webrtc.IceCandidate +import org.webrtc.PeerConnection.IceServer +import org.webrtc.SessionDescription + +/** + * Struct holding the connection parameters of an AppRTC room. + */ +data class RoomConnectionParameters( + val callId: String, + val matrixRoomId: String +) + +/** + * Struct holding the signaling parameters of an AppRTC room. + */ +data class SignalingParameters( + val iceServers: List, + val initiator: Boolean, + val clientId: String, + val offerSdp: SessionDescription, + val iceCandidates: List +) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServer.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServer.kt new file mode 100644 index 0000000000..fddd1c0c6a --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/TurnServer.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.api.session.call + +import com.squareup.moshi.Json +import com.squareup.moshi.JsonClass + +@JsonClass(generateAdapter = true) +data class TurnServer( + @Json(name = "username") val username: String?, + @Json(name = "password") val password: String?, + @Json(name = "uris") val uris: List?, + @Json(name = "ttl") val ttl: Int? +) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt new file mode 100644 index 0000000000..e324822617 --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/call/VoipApi.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.api.session.call + +import im.vector.matrix.android.internal.network.NetworkConstants +import retrofit2.Call +import retrofit2.http.GET + +internal interface VoipApi { + + @GET(NetworkConstants.URI_API_PREFIX_PATH_R0 + "voip/turnServer") + fun getTurnServer(): Call + +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/events/model/EventType.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/events/model/EventType.kt index a33b9e70df..801a9bb7d0 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/events/model/EventType.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/events/model/EventType.kt @@ -58,7 +58,6 @@ object EventType { const val STATE_ROOM_ENCRYPTION = "m.room.encryption" // Call Events - const val CALL_INVITE = "m.call.invite" const val CALL_CANDIDATES = "m.call.candidates" const val CALL_ANSWER = "m.call.answer" diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt index 29305d1420..44c3cfbf0b 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/call/CallInviteContent.kt @@ -21,21 +21,42 @@ import com.squareup.moshi.JsonClass @JsonClass(generateAdapter = true) data class CallInviteContent( - @Json(name = "call_id") val callId: String, - @Json(name = "version") val version: Int, - @Json(name = "lifetime") val lifetime: Int, - @Json(name = "offer") val offer: Offer + + /** + * A unique identifier for the call. + */ + @Json(name = "call_id") val callId: String?, + /** + * The session description object + */ + @Json(name = "version") val version: Int?, + /** + * The version of the VoIP specification this message adheres to. This specification is version 0. + */ + @Json(name = "lifetime") val lifetime: Int?, + /** + * The time in milliseconds that the invite is valid for. + * Once the invite age exceeds this value, clients should discard it. + * They should also no longer show the call as awaiting an answer in the UI. + */ + @Json(name = "offer") val offer: Offer? ) { @JsonClass(generateAdapter = true) data class Offer( - @Json(name = "type") val type: String, - @Json(name = "sdp") val sdp: String + /** + * The type of session description (offer, answer) + */ + @Json(name = "type") val type: String?, + /** + * The SDP text of the session description. + */ + @Json(name = "sdp") val sdp: String? ) { companion object { const val SDP_VIDEO = "m=video" } } - fun isVideo(): Boolean = offer.sdp.contains(Offer.SDP_VIDEO) + fun isVideo(): Boolean = offer?.sdp?.contains(Offer.SDP_VIDEO) == true } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/CryptoModule.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/CryptoModule.kt index 1efdffdb06..44092f4ae4 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/CryptoModule.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/CryptoModule.kt @@ -67,6 +67,7 @@ import im.vector.matrix.android.internal.crypto.tasks.DefaultEncryptEventTask import im.vector.matrix.android.internal.crypto.tasks.DefaultGetDeviceInfoTask import im.vector.matrix.android.internal.crypto.tasks.DefaultGetDevicesTask import im.vector.matrix.android.internal.crypto.tasks.DefaultInitializeCrossSigningTask +import im.vector.matrix.android.internal.crypto.tasks.DefaultSendEventTask import im.vector.matrix.android.internal.crypto.tasks.DefaultSendToDeviceTask import im.vector.matrix.android.internal.crypto.tasks.DefaultSendVerificationMessageTask import im.vector.matrix.android.internal.crypto.tasks.DefaultSetDeviceNameTask @@ -80,6 +81,7 @@ import im.vector.matrix.android.internal.crypto.tasks.EncryptEventTask import im.vector.matrix.android.internal.crypto.tasks.GetDeviceInfoTask import im.vector.matrix.android.internal.crypto.tasks.GetDevicesTask import im.vector.matrix.android.internal.crypto.tasks.InitializeCrossSigningTask +import im.vector.matrix.android.internal.crypto.tasks.SendEventTask import im.vector.matrix.android.internal.crypto.tasks.SendToDeviceTask import im.vector.matrix.android.internal.crypto.tasks.SendVerificationMessageTask import im.vector.matrix.android.internal.crypto.tasks.SetDeviceNameTask @@ -251,4 +253,7 @@ internal abstract class CryptoModule { @Binds abstract fun bindInitializeCrossSigningTask(task: DefaultInitializeCrossSigningTask): InitializeCrossSigningTask + + @Binds + abstract fun bindSendEventTask(task: DefaultSendEventTask): SendEventTask } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/tasks/SendEventTask.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/tasks/SendEventTask.kt new file mode 100644 index 0000000000..637db1790e --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/tasks/SendEventTask.kt @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package im.vector.matrix.android.internal.crypto.tasks + +import im.vector.matrix.android.api.session.crypto.CryptoService +import im.vector.matrix.android.api.session.events.model.Event +import im.vector.matrix.android.api.session.room.send.SendState +import im.vector.matrix.android.internal.network.executeRequest +import im.vector.matrix.android.internal.session.room.RoomAPI +import im.vector.matrix.android.internal.session.room.send.LocalEchoUpdater +import im.vector.matrix.android.internal.session.room.send.SendResponse +import im.vector.matrix.android.internal.task.Task +import org.greenrobot.eventbus.EventBus +import javax.inject.Inject + +internal interface SendEventTask : Task { + data class Params( + val event: Event, + val cryptoService: CryptoService? + ) +} + +internal class DefaultSendEventTask @Inject constructor( + private val localEchoUpdater: LocalEchoUpdater, + private val encryptEventTask: DefaultEncryptEventTask, + private val roomAPI: RoomAPI, + private val eventBus: EventBus) : SendEventTask { + + override suspend fun execute(params: SendEventTask.Params): String { + val event = handleEncryption(params) + val localId = event.eventId!! + + try { + localEchoUpdater.updateSendState(localId, SendState.SENDING) + val executeRequest = executeRequest(eventBus) { + apiCall = roomAPI.send( + localId, + roomId = event.roomId ?: "", + content = event.content, + eventType = event.type + ) + } + localEchoUpdater.updateSendState(localId, SendState.SENT) + return executeRequest.eventId + } catch (e: Throwable) { + localEchoUpdater.updateSendState(localId, SendState.UNDELIVERED) + throw e + } + } + + private suspend fun handleEncryption(params: SendEventTask.Params): Event { + if (params.cryptoService?.isRoomEncrypted(params.event.roomId ?: "") == true) { + try { + return encryptEventTask.execute(EncryptEventTask.Params( + params.event.roomId ?: "", + params.event, + listOf("m.relates_to"), + params.cryptoService + )) + } catch (throwable: Throwable) { + // We said it's ok to send verification request in clear + } + } + return params.event + } +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt index d62eb7b505..8b3affd82f 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt @@ -28,6 +28,7 @@ import im.vector.matrix.android.api.session.Session import im.vector.matrix.android.api.session.account.AccountService import im.vector.matrix.android.api.session.accountdata.AccountDataService import im.vector.matrix.android.api.session.cache.CacheService +import im.vector.matrix.android.api.session.call.CallService import im.vector.matrix.android.api.session.content.ContentUploadStateTracker import im.vector.matrix.android.api.session.content.ContentUrlResolver import im.vector.matrix.android.api.session.crypto.CryptoService @@ -110,7 +111,8 @@ internal class DefaultSession @Inject constructor( private val integrationManagerService: IntegrationManagerService, private val taskExecutor: TaskExecutor, private val widgetDependenciesHolder: WidgetDependenciesHolder, - private val shieldTrustUpdater: ShieldTrustUpdater) + private val shieldTrustUpdater: ShieldTrustUpdater, + private val callService: Lazy) : Session, RoomService by roomService.get(), RoomDirectoryService by roomDirectoryService.get(), @@ -245,6 +247,8 @@ internal class DefaultSession @Inject constructor( override fun integrationManagerService() = integrationManagerService + override fun callService(): CallService = callService.get() + override fun addListener(listener: Session.Listener) { sessionListeners.addListener(listener) } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt index c7afcc1d47..c670609411 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/SessionModule.kt @@ -33,6 +33,7 @@ import im.vector.matrix.android.api.crypto.MXCryptoConfig import im.vector.matrix.android.api.session.InitialSyncProgressService import im.vector.matrix.android.api.session.Session import im.vector.matrix.android.api.session.accountdata.AccountDataService +import im.vector.matrix.android.api.session.call.CallService import im.vector.matrix.android.api.session.homeserver.HomeServerCapabilitiesService import im.vector.matrix.android.api.session.securestorage.SecureStorageService import im.vector.matrix.android.api.session.securestorage.SharedSecretStorageService @@ -56,9 +57,9 @@ import im.vector.matrix.android.internal.network.NetworkCallbackStrategy import im.vector.matrix.android.internal.network.NetworkConnectivityChecker import im.vector.matrix.android.internal.network.PreferredNetworkCallbackStrategy import im.vector.matrix.android.internal.network.RetrofitFactory -import im.vector.matrix.android.internal.network.httpclient.addAccessTokenInterceptor -import im.vector.matrix.android.internal.network.token.AccessTokenProvider -import im.vector.matrix.android.internal.network.token.HomeserverAccessTokenProvider +import im.vector.matrix.android.internal.network.interceptors.CurlLoggingInterceptor +import im.vector.matrix.android.internal.session.call.CallEventObserver +import im.vector.matrix.android.internal.session.call.DefaultCallService import im.vector.matrix.android.internal.session.group.GroupSummaryUpdater import im.vector.matrix.android.internal.session.homeserver.DefaultHomeServerCapabilitiesService import im.vector.matrix.android.internal.session.room.EventRelationsAggregationUpdater @@ -245,7 +246,11 @@ internal abstract class SessionModule { @Binds @IntoSet - abstract fun bindRoomTombstoneEventLiveObserver(observer: RoomTombstoneEventLiveObserver): LiveEntityObserver + abstract fun bindCallEventObserver(callEventObserver: CallEventObserver): LiveEntityObserver + + @Binds + @IntoSet + abstract fun bindRoomTombstoneEventLiveObserver(roomTombstoneEventLiveObserver: RoomTombstoneEventLiveObserver): LiveEntityObserver @Binds @IntoSet @@ -269,4 +274,7 @@ internal abstract class SessionModule { @Binds abstract fun bindSharedSecretStorageService(service: DefaultSharedSecretStorageService): SharedSecretStorageService + + @Binds + abstract fun bindCallService(service:DefaultCallService): CallService } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt new file mode 100644 index 0000000000..61e6087737 --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventObserver.kt @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.internal.session.call + +import com.zhuinden.monarchy.Monarchy +import im.vector.matrix.android.api.session.events.model.EventType +import im.vector.matrix.android.internal.database.RealmLiveEntityObserver +import im.vector.matrix.android.internal.database.mapper.asDomain +import im.vector.matrix.android.internal.database.model.EventEntity +import im.vector.matrix.android.internal.database.query.whereTypes +import im.vector.matrix.android.internal.di.SessionDatabase +import im.vector.matrix.android.internal.di.UserId +import im.vector.matrix.android.internal.session.room.EventRelationsAggregationTask +import io.realm.OrderedCollectionChangeSet +import io.realm.RealmConfiguration +import io.realm.RealmResults +import kotlinx.coroutines.launch +import timber.log.Timber +import javax.inject.Inject + +internal class CallEventObserver @Inject constructor( + @SessionDatabase realmConfiguration: RealmConfiguration, + @UserId private val userId: String, + private val task: CallEventsObserverTask) : + RealmLiveEntityObserver(realmConfiguration) { + + override val query = Monarchy.Query { + EventEntity.whereTypes(it, listOf( + EventType.CALL_ANSWER, + EventType.CALL_CANDIDATES, + EventType.CALL_INVITE, + EventType.CALL_HANGUP, + EventType.ENCRYPTED) + ) + } + + override fun onChange(results: RealmResults, changeSet: OrderedCollectionChangeSet) { + Timber.v("EventRelationsAggregationUpdater called with ${changeSet.insertions.size} insertions") + + val insertedDomains = changeSet.insertions + .asSequence() + .mapNotNull { results[it]?.asDomain() } + .toList() + + val params = CallEventsObserverTask.Params( + insertedDomains, + userId + ) + observerScope.launch { + task.execute(params) + } + } +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt new file mode 100644 index 0000000000..2aeabc09f4 --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/CallEventsObserverTask.kt @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.internal.session.call + +import com.zhuinden.monarchy.Monarchy +import im.vector.matrix.android.api.session.crypto.CryptoService +import im.vector.matrix.android.api.session.crypto.MXCryptoError +import im.vector.matrix.android.api.session.events.model.Event +import im.vector.matrix.android.api.session.events.model.EventType +import im.vector.matrix.android.internal.crypto.algorithms.olm.OlmDecryptionResult +import im.vector.matrix.android.internal.task.Task +import im.vector.matrix.android.internal.util.awaitTransaction +import io.realm.Realm +import timber.log.Timber +import javax.inject.Inject + +internal interface CallEventsObserverTask : Task { + + data class Params( + val events: List, + val userId: String + ) +} + +internal class DefaultCallEventsObserverTask @Inject constructor( + private val monarchy: Monarchy, + private val cryptoService: CryptoService, + private val callService: DefaultCallService) : CallEventsObserverTask { + + override suspend fun execute(params: CallEventsObserverTask.Params) { + val events = params.events + val userId = params.userId + monarchy.awaitTransaction { realm -> + Timber.v(">>> DefaultCallEventsObserverTask[${params.hashCode()}] called with ${events.size} events") + update(realm, events, userId) + Timber.v("<<< DefaultCallEventsObserverTask[${params.hashCode()}] finished") + } + } + + private fun update(realm: Realm, events: List, userId: String) { + events.forEach { event -> + event.roomId ?: return@forEach Unit.also { + Timber.w("Event with no room id ${event.eventId}") + } + decryptIfNeeded(event) + when (event.getClearType()) { + EventType.CALL_INVITE, + EventType.CALL_CANDIDATES, + EventType.CALL_HANGUP, + EventType.CALL_ANSWER -> { + callService.onCallEvent(event) + } + } + } + Timber.v("$realm : $userId") + } + + private fun decryptIfNeeded(event: Event) { + if (event.isEncrypted() && event.mxDecryptionResult == null) { + try { + val result = cryptoService.decryptEvent(event, event.roomId ?: "") + event.mxDecryptionResult = OlmDecryptionResult( + payload = result.clearEvent, + senderKey = result.senderCurve25519Key, + keysClaimed = result.claimedEd25519Key?.let { k -> mapOf("ed25519" to k) }, + forwardingCurve25519KeyChain = result.forwardingCurve25519KeyChain + ) + } catch (e: MXCryptoError) { + Timber.v("Call service: Failed to decrypt event") + // TODO -> we should keep track of this and retry, or aggregation will be broken + } + } + } +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt new file mode 100644 index 0000000000..ab30a61ebf --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/call/DefaultCallService.kt @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.internal.session.call + +import im.vector.matrix.android.api.MatrixCallback +import im.vector.matrix.android.api.extensions.tryThis +import im.vector.matrix.android.api.session.call.CallService +import im.vector.matrix.android.api.session.call.CallsListener +import im.vector.matrix.android.api.session.call.TurnServer +import im.vector.matrix.android.api.session.events.model.Content +import im.vector.matrix.android.api.session.events.model.Event +import im.vector.matrix.android.api.session.events.model.EventType +import im.vector.matrix.android.api.session.events.model.LocalEcho +import im.vector.matrix.android.api.session.events.model.UnsignedData +import im.vector.matrix.android.api.session.events.model.toContent +import im.vector.matrix.android.api.session.events.model.toModel +import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent +import im.vector.matrix.android.api.session.room.model.call.CallCandidatesContent +import im.vector.matrix.android.api.session.room.model.call.CallInviteContent +import im.vector.matrix.android.internal.di.UserId +import im.vector.matrix.android.internal.session.SessionScope +import im.vector.matrix.android.internal.session.room.send.LocalEchoEventFactory +import im.vector.matrix.android.internal.session.room.send.RoomEventSender +import org.webrtc.IceCandidate +import org.webrtc.SessionDescription +import javax.inject.Inject + +@SessionScope +internal class DefaultCallService @Inject constructor( + @UserId + private val userId: String, + private val localEchoEventFactory: LocalEchoEventFactory, + private val roomEventSender: RoomEventSender +) : CallService { + + private val callListeners = ArrayList() + + override fun getTurnServer(callback: MatrixCallback) { + TODO("not implemented") //To change body of created functions use File | Settings | File Templates. + } + + override fun isCallSupportedInRoom(roomId: String): Boolean { + TODO("not implemented") //To change body of created functions use File | Settings | File Templates. + } + + override fun sendOfferSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) { + val eventContent = CallInviteContent( + callId = callId, + version = 0, + lifetime = CALL_TIMEOUT_MS, + offer = CallInviteContent.Offer( + type = sdp.type.canonicalForm(), + sdp = sdp.description + ) + ) + + createEventAndLocalEcho(type = EventType.CALL_INVITE, roomId = roomId, content = eventContent.toContent()).let { event -> + roomEventSender.sendEvent(event) +// sendEventTask +// .configureWith( +// SendEventTask.Params(event = event, cryptoService = cryptoService) +// ) { +// this.callback = callback +// }.executeBy(taskExecutor) + } + } + + override fun sendAnswerSdp(callId: String, roomId: String, sdp: SessionDescription, callback: MatrixCallback) { + val eventContent = CallAnswerContent( + callId = callId, + version = 0, + answer = CallAnswerContent.Answer( + type = sdp.type.canonicalForm(), + sdp = sdp.description + ) + ) + + createEventAndLocalEcho(type = EventType.CALL_INVITE, roomId = roomId, content = eventContent.toContent()).let { event -> + roomEventSender.sendEvent(event) +// sendEventTask +// .configureWith( +// SendEventTask.Params(event = event, cryptoService = cryptoService) +// ) { +// this.callback = callback +// }.executeBy(taskExecutor) + } + } + + override fun sendLocalIceCandidates(callId: String, roomId: String, candidates: List) { + val eventContent = CallCandidatesContent( + callId = callId, + version = 0, + candidates = candidates.map { + CallCandidatesContent.Candidate( + sdpMid = it.sdpMid, + sdpMLineIndex = it.sdpMLineIndex.toString(), + candidate = it.sdp + ) + } + ) + createEventAndLocalEcho(type = EventType.CALL_CANDIDATES, roomId = roomId, content = eventContent.toContent()).let { event -> + roomEventSender.sendEvent(event) +// sendEventTask +// .configureWith( +// SendEventTask.Params(event = event, cryptoService = cryptoService) +// ) { +// this.callback = callback +// }.executeBy(taskExecutor) + } + } + + override fun sendLocalIceCandidateRemovals(callId: String, roomId: String, candidates: List) { + } + + override fun addCallListener(listener: CallsListener) { + if (!callListeners.contains(listener)) callListeners.add(listener) + } + + override fun removeCallListener(listener: CallsListener) { + callListeners.remove(listener) + } + + fun onCallEvent(event: Event) { + when (event.getClearType()) { + EventType.CALL_ANSWER -> { + event.getClearContent().toModel()?.let { + onCallAnswer(it) + } + } + EventType.CALL_INVITE -> { + event.getClearContent().toModel()?.let { + onCallInvite(event.roomId ?: "", it) + } + } + } + } + + private fun onCallAnswer(answer: CallAnswerContent) { + callListeners.forEach { + tryThis { + it.onCallAnswerReceived(answer) + } + } + } + + private fun onCallInvite(roomId: String, answer: CallInviteContent) { + callListeners.forEach { + tryThis { + it.onCallInviteReceived(roomId, answer) + } + } + } + + private fun createEventAndLocalEcho(localId: String = LocalEcho.createLocalEchoId(), type: String, roomId: String, content: Content): Event { + return Event( + roomId = roomId, + originServerTs = System.currentTimeMillis(), + senderId = userId, + eventId = localId, + type = type, + content = content, + unsignedData = UnsignedData(age = null, transactionId = localId) + ).also { + localEchoEventFactory.createLocalEcho(it) + } + } + + companion object { + const val CALL_TIMEOUT_MS = 120_000 + } + +// internal class PeerSignalingClientFactory @Inject constructor( +// @UserId private val userId: String, +// private val localEchoEventFactory: LocalEchoEventFactory, +// private val sendEventTask: SendEventTask, +// private val taskExecutor: TaskExecutor, +// private val cryptoService: CryptoService +// ) { +// +// fun create(roomId: String, callId: String): PeerSignalingClient { +// return RoomPeerSignalingClient( +// callID = callId, +// roomId = roomId, +// userId = userId, +// localEchoEventFactory = localEchoEventFactory, +// sendEventTask = sendEventTask, +// taskExecutor = taskExecutor, +// cryptoService = cryptoService +// ) +// } +// } +} diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomModule.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomModule.kt index 0572a37506..e2580831e6 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomModule.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomModule.kt @@ -24,6 +24,8 @@ import im.vector.matrix.android.api.session.room.RoomDirectoryService import im.vector.matrix.android.api.session.room.RoomService import im.vector.matrix.android.internal.session.DefaultFileService import im.vector.matrix.android.internal.session.SessionScope +import im.vector.matrix.android.internal.session.call.CallEventsObserverTask +import im.vector.matrix.android.internal.session.call.DefaultCallEventsObserverTask import im.vector.matrix.android.internal.session.room.alias.DefaultGetRoomIdByAliasTask import im.vector.matrix.android.internal.session.room.alias.GetRoomIdByAliasTask import im.vector.matrix.android.internal.session.room.create.CreateRoomTask @@ -201,4 +203,8 @@ internal abstract class RoomModule { @Binds abstract fun bindDeleteTagFromRoomTask(task: DefaultDeleteTagFromRoomTask): DeleteTagFromRoomTask + + // TODO is this in the correct module? + @Binds + abstract fun bindCallEventsObserverTask(task: DefaultCallEventsObserverTask): CallEventsObserverTask } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/DefaultSendService.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/DefaultSendService.kt index d60e652e12..b4593bc71b 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/DefaultSendService.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/DefaultSendService.kt @@ -58,7 +58,8 @@ internal class DefaultSendService @AssistedInject constructor( private val localEchoEventFactory: LocalEchoEventFactory, private val cryptoService: CryptoService, private val taskExecutor: TaskExecutor, - private val localEchoRepository: LocalEchoRepository + private val localEchoRepository: LocalEchoRepository, + private val roomEventSender: RoomEventSender ) : SendService { @AssistedInject.Factory @@ -111,20 +112,6 @@ internal class DefaultSendService @AssistedInject constructor( .let { sendEvent(it) } } - private fun sendEvent(event: Event): Cancelable { - // Encrypted room handling - return if (cryptoService.isRoomEncrypted(roomId)) { - Timber.v("Send event in encrypted room") - val encryptWork = createEncryptEventWork(event, true) - // Note that event will be replaced by the result of the previous work - val sendWork = createSendEventWork(event, false) - timelineSendEventWorkCommon.postSequentialWorks(roomId, encryptWork, sendWork) - } else { - val sendWork = createSendEventWork(event, true) - timelineSendEventWorkCommon.postWork(roomId, sendWork) - } - } - override fun sendMedias(attachments: List, compressBeforeSending: Boolean, roomIds: Set): Cancelable { @@ -269,6 +256,10 @@ internal class DefaultSendService @AssistedInject constructor( return cancelableBag } + private fun sendEvent(event: Event): Cancelable { + return roomEventSender.sendEvent(event) + } + private fun createLocalEcho(event: Event) { localEchoEventFactory.createLocalEcho(event) } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/RoomEventSender.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/RoomEventSender.kt new file mode 100644 index 0000000000..4d43067ceb --- /dev/null +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/send/RoomEventSender.kt @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.matrix.android.internal.session.room.send + +import androidx.work.BackoffPolicy +import androidx.work.OneTimeWorkRequest +import im.vector.matrix.android.api.session.crypto.CryptoService +import im.vector.matrix.android.api.session.events.model.Event +import im.vector.matrix.android.api.util.Cancelable +import im.vector.matrix.android.internal.di.SessionId +import im.vector.matrix.android.internal.di.WorkManagerProvider +import im.vector.matrix.android.internal.session.room.timeline.TimelineSendEventWorkCommon +import im.vector.matrix.android.internal.worker.WorkerParamsFactory +import im.vector.matrix.android.internal.worker.startChain +import timber.log.Timber +import java.util.concurrent.TimeUnit +import javax.inject.Inject + +internal class RoomEventSender @Inject constructor( + private val workManagerProvider: WorkManagerProvider, + private val timelineSendEventWorkCommon: TimelineSendEventWorkCommon, + @SessionId private val sessionId: String, + private val cryptoService: CryptoService +) { + fun sendEvent(event: Event): Cancelable { + // Encrypted room handling + return if (cryptoService.isRoomEncrypted(event.roomId ?: "")) { + Timber.v("Send event in encrypted room") + val encryptWork = createEncryptEventWork(event, true) + // Note that event will be replaced by the result of the previous work + val sendWork = createSendEventWork(event, false) + timelineSendEventWorkCommon.postSequentialWorks(event.roomId ?: "", encryptWork, sendWork) + } else { + val sendWork = createSendEventWork(event, true) + timelineSendEventWorkCommon.postWork(event.roomId ?: "", sendWork) + } + } + + private fun createEncryptEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest { + // Same parameter + val params = EncryptEventWorker.Params(sessionId, event) + val sendWorkData = WorkerParamsFactory.toData(params) + + return workManagerProvider.matrixOneTimeWorkRequestBuilder() + .setConstraints(WorkManagerProvider.workConstraints) + .setInputData(sendWorkData) + .startChain(startChain) + .setBackoffCriteria(BackoffPolicy.LINEAR, WorkManagerProvider.BACKOFF_DELAY, TimeUnit.MILLISECONDS) + .build() + } + + private fun createSendEventWork(event: Event, startChain: Boolean): OneTimeWorkRequest { + val sendContentWorkerParams = SendEventWorker.Params(sessionId, event) + val sendWorkData = WorkerParamsFactory.toData(sendContentWorkerParams) + + return timelineSendEventWorkCommon.createWork(sendWorkData, startChain) + } +} diff --git a/vector/build.gradle b/vector/build.gradle index 5830b38559..f253501177 100644 --- a/vector/build.gradle +++ b/vector/build.gradle @@ -390,6 +390,9 @@ dependencies { implementation 'com.github.BillCarsonFr:JsonViewer:0.5' + // TODO meant for development purposes only + implementation 'org.webrtc:google-webrtc:1.0.+' + // QR-code // Stick to 3.3.3 because of https://github.com/zxing/zxing/issues/1170 implementation 'com.google.zxing:core:3.3.3' diff --git a/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt b/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt index a197a6f93e..16db7b0c38 100644 --- a/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt +++ b/vector/src/debug/java/im/vector/riotx/features/debug/DebugMenuActivity.kt @@ -35,6 +35,7 @@ import im.vector.riotx.core.utils.PERMISSION_REQUEST_CODE_LAUNCH_CAMERA import im.vector.riotx.core.utils.allGranted import im.vector.riotx.core.utils.checkPermissions import im.vector.riotx.core.utils.toast +import im.vector.riotx.features.call.VectorCallActivity import im.vector.riotx.features.debug.sas.DebugSasEmojiActivity import im.vector.riotx.features.qrcode.QrCodeScannerActivity import kotlinx.android.synthetic.debug.activity_debug_menu.* @@ -183,7 +184,8 @@ class DebugMenuActivity : VectorBaseActivity() { @OnClick(R.id.debug_scan_qr_code) fun scanQRCode() { if (checkPermissions(PERMISSIONS_FOR_TAKING_PHOTO, this, PERMISSION_REQUEST_CODE_LAUNCH_CAMERA)) { - doScanQRCode() + //doScanQRCode() + startActivity(VectorCallActivity.newIntent(this, "!cyIJhOLwWgmmqreHLD:matrix.org")) } } diff --git a/vector/src/main/AndroidManifest.xml b/vector/src/main/AndroidManifest.xml index 833267483a..1cd522337e 100644 --- a/vector/src/main/AndroidManifest.xml +++ b/vector/src/main/AndroidManifest.xml @@ -8,8 +8,16 @@ + + + + + + + + @@ -186,6 +195,13 @@ android:name=".core.services.VectorSyncService" android:exported="false" /> + + + + + + diff --git a/vector/src/main/java/im/vector/riotx/VectorApplication.kt b/vector/src/main/java/im/vector/riotx/VectorApplication.kt index 2bceb38b75..a8e597dadb 100644 --- a/vector/src/main/java/im/vector/riotx/VectorApplication.kt +++ b/vector/src/main/java/im/vector/riotx/VectorApplication.kt @@ -43,6 +43,7 @@ import im.vector.riotx.core.di.HasVectorInjector import im.vector.riotx.core.di.VectorComponent import im.vector.riotx.core.extensions.configureAndStart import im.vector.riotx.core.rx.RxConfig +import im.vector.riotx.features.call.WebRtcPeerConnectionManager import im.vector.riotx.features.configuration.VectorConfiguration import im.vector.riotx.features.lifecycle.VectorActivityLifecycleCallbacks import im.vector.riotx.features.notifications.NotificationDrawerManager @@ -80,6 +81,8 @@ class VectorApplication : Application(), HasVectorInjector, MatrixConfiguration. @Inject lateinit var appStateHandler: AppStateHandler @Inject lateinit var rxConfig: RxConfig @Inject lateinit var popupAlertManager: PopupAlertManager + @Inject lateinit var webRtcPeerConnectionManager: WebRtcPeerConnectionManager + lateinit var vectorComponent: VectorComponent private var fontThreadHandler: Handler? = null @@ -122,6 +125,7 @@ class VectorApplication : Application(), HasVectorInjector, MatrixConfiguration. val lastAuthenticatedSession = authenticationService.getLastAuthenticatedSession()!! activeSessionHolder.setActiveSession(lastAuthenticatedSession) lastAuthenticatedSession.configureAndStart(applicationContext, pushRuleTriggerListener, sessionListener) + lastAuthenticatedSession.callService().addCallListener(webRtcPeerConnectionManager) } ProcessLifecycleOwner.get().lifecycle.addObserver(object : LifecycleObserver { @OnLifecycleEvent(Lifecycle.Event.ON_RESUME) diff --git a/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt b/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt index f8ed0b01c4..ce00162e5c 100644 --- a/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt +++ b/vector/src/main/java/im/vector/riotx/core/di/ScreenComponent.kt @@ -24,6 +24,7 @@ import dagger.Component import im.vector.riotx.core.error.ErrorFormatter import im.vector.riotx.core.preference.UserAvatarPreference import im.vector.riotx.features.MainActivity +import im.vector.riotx.features.call.VectorCallActivity import im.vector.riotx.features.createdirect.CreateDirectRoomActivity import im.vector.riotx.features.crypto.keysbackup.settings.KeysBackupManageActivity import im.vector.riotx.features.crypto.quads.SharedSecureStorageActivity @@ -130,6 +131,7 @@ interface ScreenComponent { fun inject(activity: InviteUsersToRoomActivity) fun inject(activity: ReviewTermsActivity) fun inject(activity: WidgetActivity) + fun inject(activity: VectorCallActivity) /* ========================================================================================== * BottomSheets diff --git a/vector/src/main/java/im/vector/riotx/core/di/VectorComponent.kt b/vector/src/main/java/im/vector/riotx/core/di/VectorComponent.kt index 5c5052cf2b..14f3019666 100644 --- a/vector/src/main/java/im/vector/riotx/core/di/VectorComponent.kt +++ b/vector/src/main/java/im/vector/riotx/core/di/VectorComponent.kt @@ -31,6 +31,7 @@ import im.vector.riotx.core.error.ErrorFormatter import im.vector.riotx.core.pushers.PushersManager import im.vector.riotx.core.utils.AssetReader import im.vector.riotx.core.utils.DimensionConverter +import im.vector.riotx.features.call.WebRtcPeerConnectionManager import im.vector.riotx.features.configuration.VectorConfiguration import im.vector.riotx.features.crypto.keysrequest.KeyRequestHandler import im.vector.riotx.features.crypto.verification.IncomingVerificationRequestHandler @@ -134,6 +135,8 @@ interface VectorComponent { fun reAuthHelper(): ReAuthHelper + fun webRtcPeerConnectionManager(): WebRtcPeerConnectionManager + @Component.Factory interface Factory { fun create(@BindsInstance context: Context): VectorComponent diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallConnection.kt b/vector/src/main/java/im/vector/riotx/features/call/CallConnection.kt new file mode 100644 index 0000000000..5f1cd81383 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/CallConnection.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import android.content.Context +import android.os.Build +import android.telecom.Connection +import androidx.annotation.RequiresApi + +@RequiresApi(Build.VERSION_CODES.M) class CallConnection( + private val context: Context, + private val roomId: String, + private val callId: String +) : Connection() { + + /** + * The telecom subsystem calls this method when you add a new incoming call and your app should show its incoming call UI. + */ + override fun onShowIncomingCallUi() { + VectorCallActivity.newIntent(context, roomId).let { + context.startActivity(it) + } + } + + override fun onAnswer() { + super.onAnswer() + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/CallFragment.java b/vector/src/main/java/im/vector/riotx/features/call/CallFragment.java new file mode 100644 index 0000000000..750c7b6416 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/CallFragment.java @@ -0,0 +1,133 @@ +///* +// * Copyright (c) 2020 New Vector Ltd +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//package im.vector.riotx.features.call; +// +//import android.os.Bundle; +//import android.view.LayoutInflater; +//import android.view.View; +//import android.view.ViewGroup; +//import android.widget.ImageButton; +//import android.widget.SeekBar; +//import android.widget.TextView; +//import org.webrtc.RendererCommon.ScalingType; +// +//import androidx.fragment.app.Fragment; +// +//import im.vector.riotx.R; +// +///** +// * Fragment for call control. +// */ +//public class CallFragment extends Fragment { +// private TextView contactView; +// private ImageButton cameraSwitchButton; +// private ImageButton videoScalingButton; +// private ImageButton toggleMuteButton; +// private TextView captureFormatText; +// private SeekBar captureFormatSlider; +// private OnCallEvents callEvents; +// private ScalingType scalingType; +// private boolean videoCallEnabled = true; +// /** +// * Call control interface for container activity. +// */ +// public interface OnCallEvents { +// void onCallHangUp(); +// void onCameraSwitch(); +// void onVideoScalingSwitch(ScalingType scalingType); +// void onCaptureFormatChange(int width, int height, int framerate); +// boolean onToggleMic(); +// } +// @Override +// public View onCreateView( +// LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) { +// View controlView = inflater.inflate(R.layout.fragment_call, container, false); +// // Create UI controls. +// contactView = controlView.findViewById(R.id.contact_name_call); +// ImageButton disconnectButton = controlView.findViewById(R.id.button_call_disconnect); +// cameraSwitchButton = controlView.findViewById(R.id.button_call_switch_camera); +// videoScalingButton = controlView.findViewById(R.id.button_call_scaling_mode); +// toggleMuteButton = controlView.findViewById(R.id.button_call_toggle_mic); +// captureFormatText = controlView.findViewById(R.id.capture_format_text_call); +// captureFormatSlider = controlView.findViewById(R.id.capture_format_slider_call); +// // Add buttons click events. +// disconnectButton.setOnClickListener(new View.OnClickListener() { +// @Override +// public void onClick(View view) { +// callEvents.onCallHangUp(); +// } +// }); +// cameraSwitchButton.setOnClickListener(new View.OnClickListener() { +// @Override +// public void onClick(View view) { +// callEvents.onCameraSwitch(); +// } +// }); +// videoScalingButton.setOnClickListener(new View.OnClickListener() { +// @Override +// public void onClick(View view) { +// if (scalingType == ScalingType.SCALE_ASPECT_FILL) { +// videoScalingButton.setBackgroundResource(R.drawable.ic_action_full_screen); +// scalingType = ScalingType.SCALE_ASPECT_FIT; +// } else { +// videoScalingButton.setBackgroundResource(R.drawable.ic_action_return_from_full_screen); +// scalingType = ScalingType.SCALE_ASPECT_FILL; +// } +// callEvents.onVideoScalingSwitch(scalingType); +// } +// }); +// scalingType = ScalingType.SCALE_ASPECT_FILL; +// toggleMuteButton.setOnClickListener(new View.OnClickListener() { +// @Override +// public void onClick(View view) { +// boolean enabled = callEvents.onToggleMic(); +// toggleMuteButton.setAlpha(enabled ? 1.0f : 0.3f); +// } +// }); +// return controlView; +// } +// @Override +// public void onStart() { +// super.onStart(); +// boolean captureSliderEnabled = false; +// Bundle args = getArguments(); +// if (args != null) { +// String contactName = args.getString(CallActivity.EXTRA_ROOMID); +// contactView.setText(contactName); +// videoCallEnabled = args.getBoolean(CallActivity.EXTRA_VIDEO_CALL, true); +// captureSliderEnabled = videoCallEnabled +// && args.getBoolean(CallActivity.EXTRA_VIDEO_CAPTUREQUALITYSLIDER_ENABLED, false); +// } +// if (!videoCallEnabled) { +// cameraSwitchButton.setVisibility(View.INVISIBLE); +// } +// if (captureSliderEnabled) { +// captureFormatSlider.setOnSeekBarChangeListener( +// new CaptureQualityController(captureFormatText, callEvents)); +// } else { +// captureFormatText.setVisibility(View.GONE); +// captureFormatSlider.setVisibility(View.GONE); +// } +// } +// // TODO(sakal): Replace with onAttach(Context) once we only support API level 23+. +// @SuppressWarnings("deprecation") +// @Override +// public void onAttach(Activity activity) { +// super.onAttach(activity); +// callEvents = (OnCallEvents) activity; +// } +//} diff --git a/vector/src/main/java/im/vector/riotx/features/call/PeerConnectionClient.java b/vector/src/main/java/im/vector/riotx/features/call/PeerConnectionClient.java new file mode 100644 index 0000000000..ee2594fd1a --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/PeerConnectionClient.java @@ -0,0 +1,1374 @@ +///* +// * Copyright (c) 2020 New Vector Ltd +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//package im.vector.riotx.features.call; +// +//import android.content.Context; +//import android.os.Environment; +//import android.os.ParcelFileDescriptor; +//import android.util.Log; +// +//import org.appspot.apprtc.AppRTCClient.SignalingParameters; +//import org.webrtc.AudioSource; +//import org.webrtc.AudioTrack; +//import org.webrtc.CameraVideoCapturer; +//import org.webrtc.DataChannel; +//import org.webrtc.EglBase; +//import org.webrtc.IceCandidate; +//import org.webrtc.Logging; +//import org.webrtc.MediaConstraints; +//import org.webrtc.MediaStream; +//import org.webrtc.PeerConnection; +//import org.webrtc.PeerConnection.IceConnectionState; +//import org.webrtc.PeerConnectionFactory; +//import org.webrtc.RtpParameters; +//import org.webrtc.RtpReceiver; +//import org.webrtc.RtpSender; +//import org.webrtc.SdpObserver; +//import org.webrtc.SessionDescription; +//import org.webrtc.StatsObserver; +//import org.webrtc.StatsReport; +//import org.webrtc.VideoCapturer; +//import org.webrtc.VideoRenderer; +//import org.webrtc.VideoSink; +//import org.webrtc.VideoSource; +//import org.webrtc.VideoTrack; +//import org.webrtc.voiceengine.WebRtcAudioManager; +//import org.webrtc.voiceengine.WebRtcAudioRecord; +//import org.webrtc.voiceengine.WebRtcAudioRecord.AudioRecordStartErrorCode; +//import org.webrtc.voiceengine.WebRtcAudioRecord.WebRtcAudioRecordErrorCallback; +//import org.webrtc.voiceengine.WebRtcAudioTrack; +//import org.webrtc.voiceengine.WebRtcAudioTrack.WebRtcAudioTrackErrorCallback; +//import org.webrtc.voiceengine.WebRtcAudioUtils; +// +//import java.io.File; +//import java.io.IOException; +//import java.nio.ByteBuffer; +//import java.util.ArrayList; +//import java.util.Arrays; +//import java.util.Collections; +//import java.util.EnumSet; +//import java.util.Iterator; +//import java.util.LinkedList; +//import java.util.List; +//import java.util.Timer; +//import java.util.TimerTask; +//import java.util.concurrent.ExecutorService; +//import java.util.concurrent.Executors; +//import java.util.regex.Matcher; +//import java.util.regex.Pattern; +// +///** +// * Peer connection client implementation. +// * +// *

All public methods are routed to local looper thread. +// * All PeerConnectionEvents callbacks are invoked from the same looper thread. +// * This class is a singleton. +// */ +//public class PeerConnectionClient { +// public static final String VIDEO_TRACK_ID = "ARDAMSv0"; +// public static final String AUDIO_TRACK_ID = "ARDAMSa0"; +// public static final String VIDEO_TRACK_TYPE = "video"; +// private static final String TAG = "PCRTCClient"; +// private static final String VIDEO_CODEC_VP8 = "VP8"; +// private static final String VIDEO_CODEC_VP9 = "VP9"; +// private static final String VIDEO_CODEC_H264 = "H264"; +// private static final String VIDEO_CODEC_H264_BASELINE = "H264 Baseline"; +// private static final String VIDEO_CODEC_H264_HIGH = "H264 High"; +// private static final String AUDIO_CODEC_OPUS = "opus"; +// private static final String AUDIO_CODEC_ISAC = "ISAC"; +// private static final String VIDEO_CODEC_PARAM_START_BITRATE = "x-google-start-bitrate"; +// private static final String VIDEO_FLEXFEC_FIELDTRIAL = +// "WebRTC-FlexFEC-03-Advertised/Enabled/WebRTC-FlexFEC-03/Enabled/"; +// private static final String VIDEO_VP8_INTEL_HW_ENCODER_FIELDTRIAL = "WebRTC-IntelVP8/Enabled/"; +// private static final String VIDEO_H264_HIGH_PROFILE_FIELDTRIAL = +// "WebRTC-H264HighProfile/Enabled/"; +// private static final String DISABLE_WEBRTC_AGC_FIELDTRIAL = +// "WebRTC-Audio-MinimizeResamplingOnMobile/Enabled/"; +// private static final String AUDIO_CODEC_PARAM_BITRATE = "maxaveragebitrate"; +// private static final String AUDIO_ECHO_CANCELLATION_CONSTRAINT = "googEchoCancellation"; +// private static final String AUDIO_AUTO_GAIN_CONTROL_CONSTRAINT = "googAutoGainControl"; +// private static final String AUDIO_HIGH_PASS_FILTER_CONSTRAINT = "googHighpassFilter"; +// private static final String AUDIO_NOISE_SUPPRESSION_CONSTRAINT = "googNoiseSuppression"; +// private static final String AUDIO_LEVEL_CONTROL_CONSTRAINT = "levelControl"; +// private static final String DTLS_SRTP_KEY_AGREEMENT_CONSTRAINT = "DtlsSrtpKeyAgreement"; +// private static final int HD_VIDEO_WIDTH = 1280; +// private static final int HD_VIDEO_HEIGHT = 720; +// private static final int BPS_IN_KBPS = 1000; +// +// private static final PeerConnectionClient instance = new PeerConnectionClient(); +// private final PCObserver pcObserver = new PCObserver(); +// private final SDPObserver sdpObserver = new SDPObserver(); +// private final ExecutorService executor; +// +// private PeerConnectionFactory factory; +// private PeerConnection peerConnection; +// PeerConnectionFactory.Options options = null; +// private AudioSource audioSource; +// private VideoSource videoSource; +// private boolean videoCallEnabled; +// private boolean preferIsac; +// private String preferredVideoCodec; +// private boolean videoCapturerStopped; +// private boolean isError; +// private Timer statsTimer; +// private VideoSink.Callbacks localRender; +// private List remoteRenders; +// private SignalingParameters signalingParameters; +// private MediaConstraints pcConstraints; +// private int videoWidth; +// private int videoHeight; +// private int videoFps; +// private MediaConstraints audioConstraints; +// private ParcelFileDescriptor aecDumpFileDescriptor; +// private MediaConstraints sdpMediaConstraints; +// private PeerConnectionParameters peerConnectionParameters; +// // Queued remote ICE candidates are consumed only after both local and +// // remote descriptions are set. Similarly local ICE candidates are sent to +// // remote peer after both local and remote description are set. +// private LinkedList queuedRemoteCandidates; +// private PeerConnectionEvents events; +// private boolean isInitiator; +// private SessionDescription localSdp; // either offer or answer SDP +// private MediaStream mediaStream; +// private VideoCapturer videoCapturer; +// // enableVideo is set to true if video should be rendered and sent. +// private boolean renderVideo; +// private VideoTrack localVideoTrack; +// private VideoTrack remoteVideoTrack; +// private RtpSender localVideoSender; +// // enableAudio is set to true if audio should be sent. +// private boolean enableAudio; +// private AudioTrack localAudioTrack; +// private DataChannel dataChannel; +// private boolean dataChannelEnabled; +// +// /** +// * Peer connection parameters. +// */ +// public static class DataChannelParameters { +// public final boolean ordered; +// public final int maxRetransmitTimeMs; +// public final int maxRetransmits; +// public final String protocol; +// public final boolean negotiated; +// public final int id; +// +// public DataChannelParameters(boolean ordered, int maxRetransmitTimeMs, int maxRetransmits, +// String protocol, boolean negotiated, int id) { +// this.ordered = ordered; +// this.maxRetransmitTimeMs = maxRetransmitTimeMs; +// this.maxRetransmits = maxRetransmits; +// this.protocol = protocol; +// this.negotiated = negotiated; +// this.id = id; +// } +// } +// +// /** +// * Peer connection parameters. +// */ +// public static class PeerConnectionParameters { +// public final boolean videoCallEnabled; +// public final boolean loopback; +// public final boolean tracing; +// public final int videoWidth; +// public final int videoHeight; +// public final int videoFps; +// public final int videoMaxBitrate; +// public final String videoCodec; +// public final boolean videoCodecHwAcceleration; +// public final boolean videoFlexfecEnabled; +// public final int audioStartBitrate; +// public final String audioCodec; +// public final boolean noAudioProcessing; +// public final boolean aecDump; +// public final boolean useOpenSLES; +// public final boolean disableBuiltInAEC; +// public final boolean disableBuiltInAGC; +// public final boolean disableBuiltInNS; +// public final boolean enableLevelControl; +// public final boolean disableWebRtcAGCAndHPF; +// private final DataChannelParameters dataChannelParameters; +// +// public PeerConnectionParameters(boolean videoCallEnabled, boolean loopback, boolean tracing, +// int videoWidth, int videoHeight, int videoFps, int videoMaxBitrate, String videoCodec, +// boolean videoCodecHwAcceleration, boolean videoFlexfecEnabled, int audioStartBitrate, +// String audioCodec, boolean noAudioProcessing, boolean aecDump, boolean useOpenSLES, +// boolean disableBuiltInAEC, boolean disableBuiltInAGC, boolean disableBuiltInNS, +// boolean enableLevelControl, boolean disableWebRtcAGCAndHPF) { +// this(videoCallEnabled, loopback, tracing, videoWidth, videoHeight, videoFps, videoMaxBitrate, +// videoCodec, videoCodecHwAcceleration, videoFlexfecEnabled, audioStartBitrate, audioCodec, +// noAudioProcessing, aecDump, useOpenSLES, disableBuiltInAEC, disableBuiltInAGC, +// disableBuiltInNS, enableLevelControl, disableWebRtcAGCAndHPF, null); +// } +// +// public PeerConnectionParameters(boolean videoCallEnabled, boolean loopback, boolean tracing, +// int videoWidth, int videoHeight, int videoFps, int videoMaxBitrate, String videoCodec, +// boolean videoCodecHwAcceleration, boolean videoFlexfecEnabled, int audioStartBitrate, +// String audioCodec, boolean noAudioProcessing, boolean aecDump, boolean useOpenSLES, +// boolean disableBuiltInAEC, boolean disableBuiltInAGC, boolean disableBuiltInNS, +// boolean enableLevelControl, boolean disableWebRtcAGCAndHPF, +// DataChannelParameters dataChannelParameters) { +// this.videoCallEnabled = videoCallEnabled; +// this.loopback = loopback; +// this.tracing = tracing; +// this.videoWidth = videoWidth; +// this.videoHeight = videoHeight; +// this.videoFps = videoFps; +// this.videoMaxBitrate = videoMaxBitrate; +// this.videoCodec = videoCodec; +// this.videoFlexfecEnabled = videoFlexfecEnabled; +// this.videoCodecHwAcceleration = videoCodecHwAcceleration; +// this.audioStartBitrate = audioStartBitrate; +// this.audioCodec = audioCodec; +// this.noAudioProcessing = noAudioProcessing; +// this.aecDump = aecDump; +// this.useOpenSLES = useOpenSLES; +// this.disableBuiltInAEC = disableBuiltInAEC; +// this.disableBuiltInAGC = disableBuiltInAGC; +// this.disableBuiltInNS = disableBuiltInNS; +// this.enableLevelControl = enableLevelControl; +// this.disableWebRtcAGCAndHPF = disableWebRtcAGCAndHPF; +// this.dataChannelParameters = dataChannelParameters; +// } +// } +// +// /** +// * Peer connection events. +// */ +// public interface PeerConnectionEvents { +// /** +// * Callback fired once local SDP is created and set. +// */ +// void onLocalDescription(final SessionDescription sdp); +// +// /** +// * Callback fired once local Ice candidate is generated. +// */ +// void onIceCandidate(final IceCandidate candidate); +// +// /** +// * Callback fired once local ICE candidates are removed. +// */ +// void onIceCandidatesRemoved(final IceCandidate[] candidates); +// +// /** +// * Callback fired once connection is established (IceConnectionState is +// * CONNECTED). +// */ +// void onIceConnected(); +// +// /** +// * Callback fired once connection is closed (IceConnectionState is +// * DISCONNECTED). +// */ +// void onIceDisconnected(); +// +// /** +// * Callback fired once peer connection is closed. +// */ +// void onPeerConnectionClosed(); +// +// /** +// * Callback fired once peer connection statistics is ready. +// */ +// void onPeerConnectionStatsReady(final StatsReport[] reports); +// +// /** +// * Callback fired once peer connection error happened. +// */ +// void onPeerConnectionError(final String description); +// } +// +// private PeerConnectionClient() { +// // Executor thread is started once in private ctor and is used for all +// // peer connection API calls to ensure new peer connection factory is +// // created on the same thread as previously destroyed factory. +// executor = Executors.newSingleThreadExecutor(); +// } +// +// public static PeerConnectionClient getInstance() { +// return instance; +// } +// +// public void setPeerConnectionFactoryOptions(PeerConnectionFactory.Options options) { +// this.options = options; +// } +// +// public void createPeerConnectionFactory(final Context context, +// final PeerConnectionParameters peerConnectionParameters, final PeerConnectionEvents events) { +// this.peerConnectionParameters = peerConnectionParameters; +// this.events = events; +// videoCallEnabled = peerConnectionParameters.videoCallEnabled; +// dataChannelEnabled = peerConnectionParameters.dataChannelParameters != null; +// // Reset variables to initial states. +// factory = null; +// peerConnection = null; +// preferIsac = false; +// videoCapturerStopped = false; +// isError = false; +// queuedRemoteCandidates = null; +// localSdp = null; // either offer or answer SDP +// mediaStream = null; +// videoCapturer = null; +// renderVideo = true; +// localVideoTrack = null; +// remoteVideoTrack = null; +// localVideoSender = null; +// enableAudio = true; +// localAudioTrack = null; +// statsTimer = new Timer(); +// +// executor.execute(new Runnable() { +// @Override +// public void run() { +// createPeerConnectionFactoryInternal(context); +// } +// }); +// } +// +// public void createPeerConnection(final EglBase.Context renderEGLContext, +// final VideoRenderer.Callbacks localRender, final VideoRenderer.Callbacks remoteRender, +// final VideoCapturer videoCapturer, final SignalingParameters signalingParameters) { +// createPeerConnection(renderEGLContext, localRender, Collections.singletonList(remoteRender), +// videoCapturer, signalingParameters); +// } +// +// public void createPeerConnection(final EglBase.Context renderEGLContext, +// final VideoRenderer.Callbacks localRender, final List remoteRenders, +// final VideoCapturer videoCapturer, final SignalingParameters signalingParameters) { +// if (peerConnectionParameters == null) { +// Log.e(TAG, "Creating peer connection without initializing factory."); +// return; +// } +// this.localRender = localRender; +// this.remoteRenders = remoteRenders; +// this.videoCapturer = videoCapturer; +// this.signalingParameters = signalingParameters; +// executor.execute(new Runnable() { +// @Override +// public void run() { +// try { +// createMediaConstraintsInternal(); +// createPeerConnectionInternal(renderEGLContext); +// } catch (Exception e) { +// reportError("Failed to create peer connection: " + e.getMessage()); +// throw e; +// } +// } +// }); +// } +// +// public void close() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// closeInternal(); +// } +// }); +// } +// +// public boolean isVideoCallEnabled() { +// return videoCallEnabled; +// } +// +// private void createPeerConnectionFactoryInternal(Context context) { +// PeerConnectionFactory.initializeInternalTracer(); +// if (peerConnectionParameters.tracing) { +// PeerConnectionFactory.startInternalTracingCapture( +// Environment.getExternalStorageDirectory().getAbsolutePath() + File.separator +// + "webrtc-trace.txt"); +// } +// Log.d(TAG, +// "Create peer connection factory. Use video: " + peerConnectionParameters.videoCallEnabled); +// isError = false; +// +// // Initialize field trials. +// String fieldTrials = ""; +// if (peerConnectionParameters.videoFlexfecEnabled) { +// fieldTrials += VIDEO_FLEXFEC_FIELDTRIAL; +// Log.d(TAG, "Enable FlexFEC field trial."); +// } +// fieldTrials += VIDEO_VP8_INTEL_HW_ENCODER_FIELDTRIAL; +// if (peerConnectionParameters.disableWebRtcAGCAndHPF) { +// fieldTrials += DISABLE_WEBRTC_AGC_FIELDTRIAL; +// Log.d(TAG, "Disable WebRTC AGC field trial."); +// } +// +// // Check preferred video codec. +// preferredVideoCodec = VIDEO_CODEC_VP8; +// if (videoCallEnabled && peerConnectionParameters.videoCodec != null) { +// switch (peerConnectionParameters.videoCodec) { +// case VIDEO_CODEC_VP8: +// preferredVideoCodec = VIDEO_CODEC_VP8; +// break; +// case VIDEO_CODEC_VP9: +// preferredVideoCodec = VIDEO_CODEC_VP9; +// break; +// case VIDEO_CODEC_H264_BASELINE: +// preferredVideoCodec = VIDEO_CODEC_H264; +// break; +// case VIDEO_CODEC_H264_HIGH: +// // TODO(magjed): Strip High from SDP when selecting Baseline instead of using field trial. +// fieldTrials += VIDEO_H264_HIGH_PROFILE_FIELDTRIAL; +// preferredVideoCodec = VIDEO_CODEC_H264; +// break; +// default: +// preferredVideoCodec = VIDEO_CODEC_VP8; +// } +// } +// Log.d(TAG, "Preferred video codec: " + preferredVideoCodec); +// PeerConnectionFactory.initializeFieldTrials(fieldTrials); +// Log.d(TAG, "Field trials: " + fieldTrials); +// +// // Check if ISAC is used by default. +// preferIsac = peerConnectionParameters.audioCodec != null +// && peerConnectionParameters.audioCodec.equals(AUDIO_CODEC_ISAC); +// +// // Enable/disable OpenSL ES playback. +// if (!peerConnectionParameters.useOpenSLES) { +// Log.d(TAG, "Disable OpenSL ES audio even if device supports it"); +// WebRtcAudioManager.setBlacklistDeviceForOpenSLESUsage(true /* enable */); +// } else { +// Log.d(TAG, "Allow OpenSL ES audio if device supports it"); +// WebRtcAudioManager.setBlacklistDeviceForOpenSLESUsage(false); +// } +// +// if (peerConnectionParameters.disableBuiltInAEC) { +// Log.d(TAG, "Disable built-in AEC even if device supports it"); +// WebRtcAudioUtils.setWebRtcBasedAcousticEchoCanceler(true); +// } else { +// Log.d(TAG, "Enable built-in AEC if device supports it"); +// WebRtcAudioUtils.setWebRtcBasedAcousticEchoCanceler(false); +// } +// +// if (peerConnectionParameters.disableBuiltInAGC) { +// Log.d(TAG, "Disable built-in AGC even if device supports it"); +// WebRtcAudioUtils.setWebRtcBasedAutomaticGainControl(true); +// } else { +// Log.d(TAG, "Enable built-in AGC if device supports it"); +// WebRtcAudioUtils.setWebRtcBasedAutomaticGainControl(false); +// } +// +// if (peerConnectionParameters.disableBuiltInNS) { +// Log.d(TAG, "Disable built-in NS even if device supports it"); +// WebRtcAudioUtils.setWebRtcBasedNoiseSuppressor(true); +// } else { +// Log.d(TAG, "Enable built-in NS if device supports it"); +// WebRtcAudioUtils.setWebRtcBasedNoiseSuppressor(false); +// } +// +// // Set audio record error callbacks. +// WebRtcAudioRecord.setErrorCallback(new WebRtcAudioRecordErrorCallback() { +// @Override +// public void onWebRtcAudioRecordInitError(String errorMessage) { +// Log.e(TAG, "onWebRtcAudioRecordInitError: " + errorMessage); +// reportError(errorMessage); +// } +// +// @Override +// public void onWebRtcAudioRecordStartError( +// AudioRecordStartErrorCode errorCode, String errorMessage) { +// Log.e(TAG, "onWebRtcAudioRecordStartError: " + errorCode + ". " + errorMessage); +// reportError(errorMessage); +// } +// +// @Override +// public void onWebRtcAudioRecordError(String errorMessage) { +// Log.e(TAG, "onWebRtcAudioRecordError: " + errorMessage); +// reportError(errorMessage); +// } +// }); +// +// WebRtcAudioTrack.setErrorCallback(new WebRtcAudioTrackErrorCallback() { +// @Override +// public void onWebRtcAudioTrackInitError(String errorMessage) { +// reportError(errorMessage); +// } +// +// @Override +// public void onWebRtcAudioTrackStartError(String errorMessage) { +// reportError(errorMessage); +// } +// +// @Override +// public void onWebRtcAudioTrackError(String errorMessage) { +// reportError(errorMessage); +// } +// }); +// +// // Create peer connection factory. +// PeerConnectionFactory.initializeAndroidGlobals( +// context, peerConnectionParameters.videoCodecHwAcceleration); +// if (options != null) { +// Log.d(TAG, "Factory networkIgnoreMask option: " + options.networkIgnoreMask); +// } +// factory = new PeerConnectionFactory(options); +// Log.d(TAG, "Peer connection factory created."); +// } +// +// private void createMediaConstraintsInternal() { +// // Create peer connection constraints. +// pcConstraints = new MediaConstraints(); +// // Enable DTLS for normal calls and disable for loopback calls. +// if (peerConnectionParameters.loopback) { +// pcConstraints.optional.add( +// new MediaConstraints.KeyValuePair(DTLS_SRTP_KEY_AGREEMENT_CONSTRAINT, "false")); +// } else { +// pcConstraints.optional.add( +// new MediaConstraints.KeyValuePair(DTLS_SRTP_KEY_AGREEMENT_CONSTRAINT, "true")); +// } +// +// // Check if there is a camera on device and disable video call if not. +// if (videoCapturer == null) { +// Log.w(TAG, "No camera on device. Switch to audio only call."); +// videoCallEnabled = false; +// } +// // Create video constraints if video call is enabled. +// if (videoCallEnabled) { +// videoWidth = peerConnectionParameters.videoWidth; +// videoHeight = peerConnectionParameters.videoHeight; +// videoFps = peerConnectionParameters.videoFps; +// +// // If video resolution is not specified, default to HD. +// if (videoWidth == 0 || videoHeight == 0) { +// videoWidth = HD_VIDEO_WIDTH; +// videoHeight = HD_VIDEO_HEIGHT; +// } +// +// // If fps is not specified, default to 30. +// if (videoFps == 0) { +// videoFps = 30; +// } +// Logging.d(TAG, "Capturing format: " + videoWidth + "x" + videoHeight + "@" + videoFps); +// } +// +// // Create audio constraints. +// audioConstraints = new MediaConstraints(); +// // added for audio performance measurements +// if (peerConnectionParameters.noAudioProcessing) { +// Log.d(TAG, "Disabling audio processing"); +// audioConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair(AUDIO_ECHO_CANCELLATION_CONSTRAINT, "false")); +// audioConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair(AUDIO_AUTO_GAIN_CONTROL_CONSTRAINT, "false")); +// audioConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair(AUDIO_HIGH_PASS_FILTER_CONSTRAINT, "false")); +// audioConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair(AUDIO_NOISE_SUPPRESSION_CONSTRAINT, "false")); +// } +// if (peerConnectionParameters.enableLevelControl) { +// Log.d(TAG, "Enabling level control."); +// audioConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair(AUDIO_LEVEL_CONTROL_CONSTRAINT, "true")); +// } +// // Create SDP constraints. +// sdpMediaConstraints = new MediaConstraints(); +// sdpMediaConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")); +// if (videoCallEnabled || peerConnectionParameters.loopback) { +// sdpMediaConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")); +// } else { +// sdpMediaConstraints.mandatory.add( +// new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "false")); +// } +// } +// +// private void createPeerConnectionInternal(EglBase.Context renderEGLContext) { +// if (factory == null || isError) { +// Log.e(TAG, "Peerconnection factory is not created"); +// return; +// } +// Log.d(TAG, "Create peer connection."); +// +// Log.d(TAG, "PCConstraints: " + pcConstraints.toString()); +// queuedRemoteCandidates = new LinkedList(); +// +// if (videoCallEnabled) { +// Log.d(TAG, "EGLContext: " + renderEGLContext); +// factory.setVideoHwAccelerationOptions(renderEGLContext, renderEGLContext); +// } +// +// PeerConnection.RTCConfiguration rtcConfig = +// new PeerConnection.RTCConfiguration(signalingParameters.iceServers); +// // TCP candidates are only useful when connecting to a server that supports +// // ICE-TCP. +// rtcConfig.tcpCandidatePolicy = PeerConnection.TcpCandidatePolicy.DISABLED; +// rtcConfig.bundlePolicy = PeerConnection.BundlePolicy.MAXBUNDLE; +// rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.REQUIRE; +// rtcConfig.continualGatheringPolicy = PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY; +// // Use ECDSA encryption. +// rtcConfig.keyType = PeerConnection.KeyType.ECDSA; +// +// peerConnection = factory.createPeerConnection(rtcConfig, pcConstraints, pcObserver); +// +// if (dataChannelEnabled) { +// DataChannel.Init init = new DataChannel.Init(); +// init.ordered = peerConnectionParameters.dataChannelParameters.ordered; +// init.negotiated = peerConnectionParameters.dataChannelParameters.negotiated; +// init.maxRetransmits = peerConnectionParameters.dataChannelParameters.maxRetransmits; +// init.maxRetransmitTimeMs = peerConnectionParameters.dataChannelParameters.maxRetransmitTimeMs; +// init.id = peerConnectionParameters.dataChannelParameters.id; +// init.protocol = peerConnectionParameters.dataChannelParameters.protocol; +// dataChannel = peerConnection.createDataChannel("ApprtcDemo data", init); +// } +// isInitiator = false; +// +// // Set default WebRTC tracing and INFO libjingle logging. +// // NOTE: this _must_ happen while |factory| is alive! +// Logging.enableTracing("logcat:", EnumSet.of(Logging.TraceLevel.TRACE_DEFAULT)); +// Logging.enableLogToDebugOutput(Logging.Severity.LS_INFO); +// +// mediaStream = factory.createLocalMediaStream("ARDAMS"); +// if (videoCallEnabled) { +// mediaStream.addTrack(createVideoTrack(videoCapturer)); +// } +// +// mediaStream.addTrack(createAudioTrack()); +// peerConnection.addStream(mediaStream); +// if (videoCallEnabled) { +// findVideoSender(); +// } +// +// if (peerConnectionParameters.aecDump) { +// try { +// aecDumpFileDescriptor = +// ParcelFileDescriptor.open(new File(Environment.getExternalStorageDirectory().getPath() +// + File.separator + "Download/audio.aecdump"), +// ParcelFileDescriptor.MODE_READ_WRITE | ParcelFileDescriptor.MODE_CREATE +// | ParcelFileDescriptor.MODE_TRUNCATE); +// factory.startAecDump(aecDumpFileDescriptor.getFd(), -1); +// } catch (IOException e) { +// Log.e(TAG, "Can not open aecdump file", e); +// } +// } +// +// Log.d(TAG, "Peer connection created."); +// } +// +// private void closeInternal() { +// if (factory != null && peerConnectionParameters.aecDump) { +// factory.stopAecDump(); +// } +// Log.d(TAG, "Closing peer connection."); +// statsTimer.cancel(); +// if (dataChannel != null) { +// dataChannel.dispose(); +// dataChannel = null; +// } +// if (peerConnection != null) { +// peerConnection.dispose(); +// peerConnection = null; +// } +// Log.d(TAG, "Closing audio source."); +// if (audioSource != null) { +// audioSource.dispose(); +// audioSource = null; +// } +// Log.d(TAG, "Stopping capture."); +// if (videoCapturer != null) { +// try { +// videoCapturer.stopCapture(); +// } catch (InterruptedException e) { +// throw new RuntimeException(e); +// } +// videoCapturerStopped = true; +// videoCapturer.dispose(); +// videoCapturer = null; +// } +// Log.d(TAG, "Closing video source."); +// if (videoSource != null) { +// videoSource.dispose(); +// videoSource = null; +// } +// localRender = null; +// remoteRenders = null; +// Log.d(TAG, "Closing peer connection factory."); +// if (factory != null) { +// factory.dispose(); +// factory = null; +// } +// options = null; +// Log.d(TAG, "Closing peer connection done."); +// events.onPeerConnectionClosed(); +// PeerConnectionFactory.stopInternalTracingCapture(); +// PeerConnectionFactory.shutdownInternalTracer(); +// events = null; +// } +// +// public boolean isHDVideo() { +// if (!videoCallEnabled) { +// return false; +// } +// +// return videoWidth * videoHeight >= 1280 * 720; +// } +// +// private void getStats() { +// if (peerConnection == null || isError) { +// return; +// } +// boolean success = peerConnection.getStats(new StatsObserver() { +// @Override +// public void onComplete(final StatsReport[] reports) { +// events.onPeerConnectionStatsReady(reports); +// } +// }, null); +// if (!success) { +// Log.e(TAG, "getStats() returns false!"); +// } +// } +// +// public void enableStatsEvents(boolean enable, int periodMs) { +// if (enable) { +// try { +// statsTimer.schedule(new TimerTask() { +// @Override +// public void run() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// getStats(); +// } +// }); +// } +// }, 0, periodMs); +// } catch (Exception e) { +// Log.e(TAG, "Can not schedule statistics timer", e); +// } +// } else { +// statsTimer.cancel(); +// } +// } +// +// public void setAudioEnabled(final boolean enable) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// enableAudio = enable; +// if (localAudioTrack != null) { +// localAudioTrack.setEnabled(enableAudio); +// } +// } +// }); +// } +// +// public void setVideoEnabled(final boolean enable) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// renderVideo = enable; +// if (localVideoTrack != null) { +// localVideoTrack.setEnabled(renderVideo); +// } +// if (remoteVideoTrack != null) { +// remoteVideoTrack.setEnabled(renderVideo); +// } +// } +// }); +// } +// +// public void createOffer() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection != null && !isError) { +// Log.d(TAG, "PC Create OFFER"); +// isInitiator = true; +// peerConnection.createOffer(sdpObserver, sdpMediaConstraints); +// } +// } +// }); +// } +// +// public void createAnswer() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection != null && !isError) { +// Log.d(TAG, "PC create ANSWER"); +// isInitiator = false; +// peerConnection.createAnswer(sdpObserver, sdpMediaConstraints); +// } +// } +// }); +// } +// +// public void addRemoteIceCandidate(final IceCandidate candidate) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection != null && !isError) { +// if (queuedRemoteCandidates != null) { +// queuedRemoteCandidates.add(candidate); +// } else { +// peerConnection.addIceCandidate(candidate); +// } +// } +// } +// }); +// } +// +// public void removeRemoteIceCandidates(final IceCandidate[] candidates) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection == null || isError) { +// return; +// } +// // Drain the queued remote candidates if there is any so that +// // they are processed in the proper order. +// drainCandidates(); +// peerConnection.removeIceCandidates(candidates); +// } +// }); +// } +// +// public void setRemoteDescription(final SessionDescription sdp) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection == null || isError) { +// return; +// } +// String sdpDescription = sdp.description; +// if (preferIsac) { +// sdpDescription = preferCodec(sdpDescription, AUDIO_CODEC_ISAC, true); +// } +// if (videoCallEnabled) { +// sdpDescription = preferCodec(sdpDescription, preferredVideoCodec, false); +// } +// if (peerConnectionParameters.audioStartBitrate > 0) { +// sdpDescription = setStartBitrate( +// AUDIO_CODEC_OPUS, false, sdpDescription, peerConnectionParameters.audioStartBitrate); +// } +// Log.d(TAG, "Set remote SDP."); +// SessionDescription sdpRemote = new SessionDescription(sdp.type, sdpDescription); +// peerConnection.setRemoteDescription(sdpObserver, sdpRemote); +// } +// }); +// } +// +// public void stopVideoSource() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (videoCapturer != null && !videoCapturerStopped) { +// Log.d(TAG, "Stop video source."); +// try { +// videoCapturer.stopCapture(); +// } catch (InterruptedException e) { +// } +// videoCapturerStopped = true; +// } +// } +// }); +// } +// +// public void startVideoSource() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (videoCapturer != null && videoCapturerStopped) { +// Log.d(TAG, "Restart video source."); +// videoCapturer.startCapture(videoWidth, videoHeight, videoFps); +// videoCapturerStopped = false; +// } +// } +// }); +// } +// +// public void setVideoMaxBitrate(final Integer maxBitrateKbps) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection == null || localVideoSender == null || isError) { +// return; +// } +// Log.d(TAG, "Requested max video bitrate: " + maxBitrateKbps); +// if (localVideoSender == null) { +// Log.w(TAG, "Sender is not ready."); +// return; +// } +// +// RtpParameters parameters = localVideoSender.getParameters(); +// if (parameters.encodings.size() == 0) { +// Log.w(TAG, "RtpParameters are not ready."); +// return; +// } +// +// for (RtpParameters.Encoding encoding : parameters.encodings) { +// // Null value means no limit. +// encoding.maxBitrateBps = maxBitrateKbps == null ? null : maxBitrateKbps * BPS_IN_KBPS; +// } +// if (!localVideoSender.setParameters(parameters)) { +// Log.e(TAG, "RtpSender.setParameters failed."); +// } +// Log.d(TAG, "Configured max video bitrate to: " + maxBitrateKbps); +// } +// }); +// } +// +// private void reportError(final String errorMessage) { +// Log.e(TAG, "Peerconnection error: " + errorMessage); +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (!isError) { +// events.onPeerConnectionError(errorMessage); +// isError = true; +// } +// } +// }); +// } +// +// private AudioTrack createAudioTrack() { +// audioSource = factory.createAudioSource(audioConstraints); +// localAudioTrack = factory.createAudioTrack(AUDIO_TRACK_ID, audioSource); +// localAudioTrack.setEnabled(enableAudio); +// return localAudioTrack; +// } +//æ +// private VideoTrack createVideoTrack(VideoCapturer capturer) { +// videoSource = factory.createVideoSource(capturer); +// capturer.startCapture(videoWidth, videoHeight, videoFps); +// +// localVideoTrack = factory.createVideoTrack(VIDEO_TRACK_ID, videoSource); +// localVideoTrack.setEnabled(renderVideo); +// localVideoTrack.addRenderer(new VideoRenderer(localRender)); +// return localVideoTrack; +// } +// +// private void findVideoSender() { +// for (RtpSender sender : peerConnection.getSenders()) { +// if (sender.track() != null) { +// String trackType = sender.track().kind(); +// if (trackType.equals(VIDEO_TRACK_TYPE)) { +// Log.d(TAG, "Found video sender."); +// localVideoSender = sender; +// } +// } +// } +// } +// +// private static String setStartBitrate( +// String codec, boolean isVideoCodec, String sdpDescription, int bitrateKbps) { +// String[] lines = sdpDescription.split("\r\n"); +// int rtpmapLineIndex = -1; +// boolean sdpFormatUpdated = false; +// String codecRtpMap = null; +// // Search for codec rtpmap in format +// // a=rtpmap: / [/] +// String regex = "^a=rtpmap:(\\d+) " + codec + "(/\\d+)+[\r]?$"; +// Pattern codecPattern = Pattern.compile(regex); +// for (int i = 0; i < lines.length; i++) { +// Matcher codecMatcher = codecPattern.matcher(lines[i]); +// if (codecMatcher.matches()) { +// codecRtpMap = codecMatcher.group(1); +// rtpmapLineIndex = i; +// break; +// } +// } +// if (codecRtpMap == null) { +// Log.w(TAG, "No rtpmap for " + codec + " codec"); +// return sdpDescription; +// } +// Log.d(TAG, "Found " + codec + " rtpmap " + codecRtpMap + " at " + lines[rtpmapLineIndex]); +// +// // Check if a=fmtp string already exist in remote SDP for this codec and +// // update it with new bitrate parameter. +// regex = "^a=fmtp:" + codecRtpMap + " \\w+=\\d+.*[\r]?$"; +// codecPattern = Pattern.compile(regex); +// for (int i = 0; i < lines.length; i++) { +// Matcher codecMatcher = codecPattern.matcher(lines[i]); +// if (codecMatcher.matches()) { +// Log.d(TAG, "Found " + codec + " " + lines[i]); +// if (isVideoCodec) { +// lines[i] += "; " + VIDEO_CODEC_PARAM_START_BITRATE + "=" + bitrateKbps; +// } else { +// lines[i] += "; " + AUDIO_CODEC_PARAM_BITRATE + "=" + (bitrateKbps * 1000); +// } +// Log.d(TAG, "Update remote SDP line: " + lines[i]); +// sdpFormatUpdated = true; +// break; +// } +// } +// +// StringBuilder newSdpDescription = new StringBuilder(); +// for (int i = 0; i < lines.length; i++) { +// newSdpDescription.append(lines[i]).append("\r\n"); +// // Append new a=fmtp line if no such line exist for a codec. +// if (!sdpFormatUpdated && i == rtpmapLineIndex) { +// String bitrateSet; +// if (isVideoCodec) { +// bitrateSet = +// "a=fmtp:" + codecRtpMap + " " + VIDEO_CODEC_PARAM_START_BITRATE + "=" + bitrateKbps; +// } else { +// bitrateSet = "a=fmtp:" + codecRtpMap + " " + AUDIO_CODEC_PARAM_BITRATE + "=" +// + (bitrateKbps * 1000); +// } +// Log.d(TAG, "Add remote SDP line: " + bitrateSet); +// newSdpDescription.append(bitrateSet).append("\r\n"); +// } +// } +// return newSdpDescription.toString(); +// } +// +// /** +// * Returns the line number containing "m=audio|video", or -1 if no such line exists. +// */ +// private static int findMediaDescriptionLine(boolean isAudio, String[] sdpLines) { +// final String mediaDescription = isAudio ? "m=audio " : "m=video "; +// for (int i = 0; i < sdpLines.length; ++i) { +// if (sdpLines[i].startsWith(mediaDescription)) { +// return i; +// } +// } +// return -1; +// } +// +// private static String joinString( +// Iterable s, String delimiter, boolean delimiterAtEnd) { +// Iterator iter = s.iterator(); +// if (!iter.hasNext()) { +// return ""; +// } +// StringBuilder buffer = new StringBuilder(iter.next()); +// while (iter.hasNext()) { +// buffer.append(delimiter).append(iter.next()); +// } +// if (delimiterAtEnd) { +// buffer.append(delimiter); +// } +// return buffer.toString(); +// } +// +// private static String movePayloadTypesToFront(List preferredPayloadTypes, String mLine) { +// // The format of the media description line should be: m= ... +// final List origLineParts = Arrays.asList(mLine.split(" ")); +// if (origLineParts.size() <= 3) { +// Log.e(TAG, "Wrong SDP media description format: " + mLine); +// return null; +// } +// final List header = origLineParts.subList(0, 3); +// final List unpreferredPayloadTypes = +// new ArrayList(origLineParts.subList(3, origLineParts.size())); +// unpreferredPayloadTypes.removeAll(preferredPayloadTypes); +// // Reconstruct the line with |preferredPayloadTypes| moved to the beginning of the payload +// // types. +// final List newLineParts = new ArrayList(); +// newLineParts.addAll(header); +// newLineParts.addAll(preferredPayloadTypes); +// newLineParts.addAll(unpreferredPayloadTypes); +// return joinString(newLineParts, " ", false /* delimiterAtEnd */); +// } +// +// private static String preferCodec(String sdpDescription, String codec, boolean isAudio) { +// final String[] lines = sdpDescription.split("\r\n"); +// final int mLineIndex = findMediaDescriptionLine(isAudio, lines); +// if (mLineIndex == -1) { +// Log.w(TAG, "No mediaDescription line, so can't prefer " + codec); +// return sdpDescription; +// } +// // A list with all the payload types with name |codec|. The payload types are integers in the +// // range 96-127, but they are stored as strings here. +// final List codecPayloadTypes = new ArrayList(); +// // a=rtpmap: / [/] +// final Pattern codecPattern = Pattern.compile("^a=rtpmap:(\\d+) " + codec + "(/\\d+)+[\r]?$"); +// for (int i = 0; i < lines.length; ++i) { +// Matcher codecMatcher = codecPattern.matcher(lines[i]); +// if (codecMatcher.matches()) { +// codecPayloadTypes.add(codecMatcher.group(1)); +// } +// } +// if (codecPayloadTypes.isEmpty()) { +// Log.w(TAG, "No payload types with name " + codec); +// return sdpDescription; +// } +// +// final String newMLine = movePayloadTypesToFront(codecPayloadTypes, lines[mLineIndex]); +// if (newMLine == null) { +// return sdpDescription; +// } +// Log.d(TAG, "Change media description from: " + lines[mLineIndex] + " to " + newMLine); +// lines[mLineIndex] = newMLine; +// return joinString(Arrays.asList(lines), "\r\n", true /* delimiterAtEnd */); +// } +// +// private void drainCandidates() { +// if (queuedRemoteCandidates != null) { +// Log.d(TAG, "Add " + queuedRemoteCandidates.size() + " remote candidates"); +// for (IceCandidate candidate : queuedRemoteCandidates) { +// peerConnection.addIceCandidate(candidate); +// } +// queuedRemoteCandidates = null; +// } +// } +// +// private void switchCameraInternal() { +// if (videoCapturer instanceof CameraVideoCapturer) { +// if (!videoCallEnabled || isError || videoCapturer == null) { +// Log.e(TAG, "Failed to switch camera. Video: " + videoCallEnabled + ". Error : " + isError); +// return; // No video is sent or only one camera is available or error happened. +// } +// Log.d(TAG, "Switch camera"); +// CameraVideoCapturer cameraVideoCapturer = (CameraVideoCapturer) videoCapturer; +// cameraVideoCapturer.switchCamera(null); +// } else { +// Log.d(TAG, "Will not switch camera, video caputurer is not a camera"); +// } +// } +// +// public void switchCamera() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// switchCameraInternal(); +// } +// }); +// } +// +// public void changeCaptureFormat(final int width, final int height, final int framerate) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// changeCaptureFormatInternal(width, height, framerate); +// } +// }); +// } +// +// private void changeCaptureFormatInternal(int width, int height, int framerate) { +// if (!videoCallEnabled || isError || videoCapturer == null) { +// Log.e(TAG, +// "Failed to change capture format. Video: " + videoCallEnabled + ". Error : " + isError); +// return; +// } +// Log.d(TAG, "changeCaptureFormat: " + width + "x" + height + "@" + framerate); +// videoSource.adaptOutputFormat(width, height, framerate); +// } +// +// // Implementation detail: observe ICE & stream changes and react accordingly. +// private class PCObserver implements PeerConnection.Observer { +// @Override +// public void onIceCandidate(final IceCandidate candidate) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// events.onIceCandidate(candidate); +// } +// }); +// } +// +// @Override +// public void onIceCandidatesRemoved(final IceCandidate[] candidates) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// events.onIceCandidatesRemoved(candidates); +// } +// }); +// } +// +// @Override +// public void onSignalingChange(PeerConnection.SignalingState newState) { +// Log.d(TAG, "SignalingState: " + newState); +// } +// +// @Override +// public void onIceConnectionChange(final PeerConnection.IceConnectionState newState) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// Log.d(TAG, "IceConnectionState: " + newState); +// if (newState == IceConnectionState.CONNECTED) { +// events.onIceConnected(); +// } else if (newState == IceConnectionState.DISCONNECTED) { +// events.onIceDisconnected(); +// } else if (newState == IceConnectionState.FAILED) { +// reportError("ICE connection failed."); +// } +// } +// }); +// } +// +// @Override +// public void onIceGatheringChange(PeerConnection.IceGatheringState newState) { +// Log.d(TAG, "IceGatheringState: " + newState); +// } +// +// @Override +// public void onIceConnectionReceivingChange(boolean receiving) { +// Log.d(TAG, "IceConnectionReceiving changed to " + receiving); +// } +// +// @Override +// public void onAddStream(final MediaStream stream) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection == null || isError) { +// return; +// } +// if (stream.audioTracks.size() > 1 || stream.videoTracks.size() > 1) { +// reportError("Weird-looking stream: " + stream); +// return; +// } +// if (stream.videoTracks.size() == 1) { +// remoteVideoTrack = stream.videoTracks.get(0); +// remoteVideoTrack.setEnabled(renderVideo); +// for (VideoRenderer.Callbacks remoteRender : remoteRenders) { +// remoteVideoTrack.addRenderer(new VideoRenderer(remoteRender)); +// } +// } +// } +// }); +// } +// +// @Override +// public void onRemoveStream(final MediaStream stream) { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// remoteVideoTrack = null; +// } +// }); +// } +// +// @Override +// public void onDataChannel(final DataChannel dc) { +// Log.d(TAG, "New Data channel " + dc.label()); +// +// if (!dataChannelEnabled) +// return; +// +// dc.registerObserver(new DataChannel.Observer() { +// public void onBufferedAmountChange(long previousAmount) { +// Log.d(TAG, "Data channel buffered amount changed: " + dc.label() + ": " + dc.state()); +// } +// +// @Override +// public void onStateChange() { +// Log.d(TAG, "Data channel state changed: " + dc.label() + ": " + dc.state()); +// } +// +// @Override +// public void onMessage(final DataChannel.Buffer buffer) { +// if (buffer.binary) { +// Log.d(TAG, "Received binary msg over " + dc); +// return; +// } +// ByteBuffer data = buffer.data; +// final byte[] bytes = new byte[data.capacity()]; +// data.get(bytes); +// String strData = new String(bytes); +// Log.d(TAG, "Got msg: " + strData + " over " + dc); +// } +// }); +// } +// +// @Override +// public void onRenegotiationNeeded() { +// // No need to do anything; AppRTC follows a pre-agreed-upon +// // signaling/negotiation protocol. +// } +// +// @Override +// public void onAddTrack(final RtpReceiver receiver, final MediaStream[] mediaStreams) { +// } +// } +// +// // Implementation detail: handle offer creation/signaling and answer setting, +// // as well as adding remote ICE candidates once the answer SDP is set. +// private class SDPObserver implements SdpObserver { +// @Override +// public void onCreateSuccess(final SessionDescription origSdp) { +// if (localSdp != null) { +// reportError("Multiple SDP create."); +// return; +// } +// String sdpDescription = origSdp.description; +// if (preferIsac) { +// sdpDescription = preferCodec(sdpDescription, AUDIO_CODEC_ISAC, true); +// } +// if (videoCallEnabled) { +// sdpDescription = preferCodec(sdpDescription, preferredVideoCodec, false); +// } +// final SessionDescription sdp = new SessionDescription(origSdp.type, sdpDescription); +// localSdp = sdp; +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection != null && !isError) { +// Log.d(TAG, "Set local SDP from " + sdp.type); +// peerConnection.setLocalDescription(sdpObserver, sdp); +// } +// } +// }); +// } +// +// @Override +// public void onSetSuccess() { +// executor.execute(new Runnable() { +// @Override +// public void run() { +// if (peerConnection == null || isError) { +// return; +// } +// if (isInitiator) { +// // For offering peer connection we first create offer and set +// // local SDP, then after receiving answer set remote SDP. +// if (peerConnection.getRemoteDescription() == null) { +// // We've just set our local SDP so time to send it. +// Log.d(TAG, "Local SDP set succesfully"); +// events.onLocalDescription(localSdp); +// } else { +// // We've just set remote description, so drain remote +// // and send local ICE candidates. +// Log.d(TAG, "Remote SDP set succesfully"); +// drainCandidates(); +// } +// } else { +// // For answering peer connection we set remote SDP and then +// // create answer and set local SDP. +// if (peerConnection.getLocalDescription() != null) { +// // We've just set our local SDP so time to send it, drain +// // remote and send local ICE candidates. +// Log.d(TAG, "Local SDP set succesfully"); +// events.onLocalDescription(localSdp); +// drainCandidates(); +// } else { +// // We've just set remote SDP - do nothing for now - +// // answer will be created soon. +// Log.d(TAG, "Remote SDP set succesfully"); +// } +// } +// } +// }); +// } +// +// @Override +// public void onCreateFailure(final String error) { +// reportError("createSDP error: " + error); +// } +// +// @Override +// public void onSetFailure(final String error) { +// reportError("setSDP error: " + error); +// } +// } +//} diff --git a/vector/src/main/java/im/vector/riotx/features/call/PeerConnectionObserverAdapter.kt b/vector/src/main/java/im/vector/riotx/features/call/PeerConnectionObserverAdapter.kt new file mode 100644 index 0000000000..ffc90d47fa --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/PeerConnectionObserverAdapter.kt @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import org.webrtc.DataChannel +import org.webrtc.IceCandidate +import org.webrtc.MediaStream +import org.webrtc.PeerConnection +import org.webrtc.RtpReceiver +import timber.log.Timber + +abstract class PeerConnectionObserverAdapter : PeerConnection.Observer { + override fun onIceCandidate(p0: IceCandidate?) { + Timber.v("## VOIP onIceCandidate $p0") + } + + override fun onDataChannel(p0: DataChannel?) { + Timber.v("## VOIP onDataChannel $p0") + } + + override fun onIceConnectionReceivingChange(p0: Boolean) { + Timber.v("## VOIP onIceConnectionReceivingChange $p0") + } + + override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { + Timber.v("## VOIP onIceConnectionChange $p0") + } + + override fun onIceGatheringChange(p0: PeerConnection.IceGatheringState?) { + Timber.v("## VOIP onIceConnectionChange $p0") + } + + override fun onAddStream(mediaStream: MediaStream?) { + Timber.v("## VOIP onAddStream $mediaStream") + } + + override fun onSignalingChange(p0: PeerConnection.SignalingState?) { + Timber.v("## VOIP onSignalingChange $p0") + } + + override fun onIceCandidatesRemoved(p0: Array?) { + Timber.v("## VOIP onIceCandidatesRemoved $p0") + } + + override fun onRemoveStream(mediaStream: MediaStream?) { + Timber.v("## VOIP onRemoveStream $mediaStream") + } + + override fun onRenegotiationNeeded() { + Timber.v("## VOIP onRenegotiationNeeded") + } + + override fun onAddTrack(p0: RtpReceiver?, p1: Array?) { + Timber.v("## VOIP onAddTrack $p0 / out: $p1") + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/SdpObserverAdapter.kt b/vector/src/main/java/im/vector/riotx/features/call/SdpObserverAdapter.kt new file mode 100644 index 0000000000..0e15c97052 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/SdpObserverAdapter.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import org.webrtc.SdpObserver +import org.webrtc.SessionDescription +import timber.log.Timber + +abstract class SdpObserverAdapter : SdpObserver { + override fun onSetFailure(p0: String?) { + Timber.e("## SdpObserver: onSetFailure $p0") + } + + override fun onSetSuccess() { + Timber.v("## SdpObserver: onSetSuccess") + } + + override fun onCreateSuccess(p0: SessionDescription?) { + Timber.e("## SdpObserver: onSetFailure $p0") + } + + override fun onCreateFailure(p0: String?) { + Timber.e("## SdpObserver: onSetFailure $p0") + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt new file mode 100644 index 0000000000..c8387daa20 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallActivity.kt @@ -0,0 +1,434 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import android.app.KeyguardManager +import android.content.Context +import android.content.Intent +import android.os.Build +import android.os.Bundle +import android.os.Parcelable +import android.view.View +import android.view.Window +import android.view.WindowManager +import butterknife.BindView +import com.airbnb.mvrx.MvRx +import com.airbnb.mvrx.viewModel +import im.vector.matrix.android.api.session.call.EglUtils +import im.vector.riotx.R +import im.vector.riotx.core.di.ScreenComponent +import im.vector.riotx.core.platform.VectorBaseActivity +import im.vector.riotx.core.utils.PERMISSIONS_FOR_VIDEO_IP_CALL +import im.vector.riotx.core.utils.allGranted +import im.vector.riotx.core.utils.checkPermissions +import io.reactivex.android.schedulers.AndroidSchedulers +import io.reactivex.subjects.PublishSubject +import kotlinx.android.parcel.Parcelize +import org.webrtc.Camera1Enumerator +import org.webrtc.Camera2Enumerator +import org.webrtc.EglBase +import org.webrtc.IceCandidate +import org.webrtc.MediaStream +import org.webrtc.PeerConnection +import org.webrtc.RendererCommon +import org.webrtc.SessionDescription +import org.webrtc.SurfaceViewRenderer +import org.webrtc.VideoTrack +import java.util.concurrent.TimeUnit +import javax.inject.Inject + +@Parcelize +data class CallArgs( +// val callId: String? = null, + val roomId: String +) : Parcelable + +class VectorCallActivity : VectorBaseActivity(), WebRtcPeerConnectionManager.Listener { + + override fun getLayoutRes() = R.layout.activity_call + + override fun injectWith(injector: ScreenComponent) { + super.injectWith(injector) + injector.inject(this) + } + + private val callViewModel: VectorCallViewModel by viewModel() + + @Inject lateinit var peerConnectionManager: WebRtcPeerConnectionManager + + @Inject lateinit var viewModelFactory: VectorCallViewModel.Factory + + @BindView(R.id.pip_video_view) + lateinit var pipRenderer: SurfaceViewRenderer + + @BindView(R.id.fullscreen_video_view) + lateinit var fullscreenRenderer: SurfaceViewRenderer + + private var rootEglBase: EglBase? = null + +// private var peerConnectionFactory: PeerConnectionFactory? = null + + //private var peerConnection: PeerConnection? = null + +// private var remoteVideoTrack: VideoTrack? = null + + private val iceCandidateSource: PublishSubject = PublishSubject.create() + + override fun doBeforeSetContentView() { + // Set window styles for fullscreen-window size. Needs to be done before adding content. + requestWindowFeature(Window.FEATURE_NO_TITLE) + window.addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN or WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) { + setTurnScreenOn(true) + setShowWhenLocked(true) + getSystemService(KeyguardManager::class.java)?.requestDismissKeyguard(this, null) + } else { + @Suppress("DEPRECATION") + window.addFlags( + WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON + or WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED + or WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD + ) + } + + window.decorView.systemUiVisibility = + View.SYSTEM_UI_FLAG_HIDE_NAVIGATION or View.SYSTEM_UI_FLAG_FULLSCREEN or View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY + setContentView(R.layout.activity_call) + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + rootEglBase = EglUtils.rootEglBase ?: return Unit.also { + finish() + } + + callViewModel.viewEvents + .observe() + .observeOn(AndroidSchedulers.mainThread()) + .subscribe { + handleViewEvents(it) + } + .disposeOnDestroy() +// +// if (isFirstCreation()) { +// +// } + + if (checkPermissions(PERMISSIONS_FOR_VIDEO_IP_CALL, this, CAPTURE_PERMISSION_REQUEST_CODE, R.string.permissions_rationale_msg_camera_and_audio)) { + start() + } + peerConnectionManager.listener = this + } + + override fun onRequestPermissionsResult(requestCode: Int, permissions: Array, grantResults: IntArray) { + if (requestCode == CAPTURE_PERMISSION_REQUEST_CODE && allGranted(grantResults)) { + start() + } else { + // TODO display something + finish() + } + } + + private fun start(): Boolean { + // Init Picture in Picture renderer + pipRenderer.init(rootEglBase!!.eglBaseContext, null) + pipRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT) + + // Init Full Screen renderer + fullscreenRenderer.init(rootEglBase!!.eglBaseContext, null) + fullscreenRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT) + + + pipRenderer.setZOrderMediaOverlay(true); + pipRenderer.setEnableHardwareScaler(true /* enabled */); + fullscreenRenderer.setEnableHardwareScaler(true /* enabled */); + // Start with local feed in fullscreen and swap it to the pip when the call is connected. + //setSwappedFeeds(true /* isSwappedFeeds */); + + if (isFirstCreation()) { + peerConnectionManager.createPeerConnectionFactory() + + val cameraIterator = if (Camera2Enumerator.isSupported(this)) Camera2Enumerator(this) else Camera1Enumerator(false) + val frontCamera = cameraIterator.deviceNames + ?.firstOrNull { cameraIterator.isFrontFacing(it) } + ?: cameraIterator.deviceNames?.first() + ?: return true + val videoCapturer = cameraIterator.createCapturer(frontCamera, null) + + val iceServers = ArrayList().apply { + listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach { + add( + PeerConnection.IceServer.builder(it) + .setUsername("xxxxx") + .setPassword("xxxxx") + .createIceServer() + ) + } + } + + peerConnectionManager.createPeerConnection(videoCapturer, iceServers) + peerConnectionManager.startCall() + } +// PeerConnectionFactory.initialize(PeerConnectionFactory +// .InitializationOptions.builder(applicationContext) +// .createInitializationOptions() +// ) + +// val options = PeerConnectionFactory.Options() +// val defaultVideoEncoderFactory = DefaultVideoEncoderFactory( +// rootEglBase!!.eglBaseContext, /* enableIntelVp8Encoder */ +// true, /* enableH264HighProfile */ +// true) +// val defaultVideoDecoderFactory = DefaultVideoDecoderFactory(rootEglBase!!.eglBaseContext) +// +// peerConnectionFactory = PeerConnectionFactory.builder() +// .setOptions(options) +// .setVideoEncoderFactory(defaultVideoEncoderFactory) +// .setVideoDecoderFactory(defaultVideoDecoderFactory) +// .createPeerConnectionFactory() + +// val cameraIterator = if (Camera2Enumerator.isSupported(this)) Camera2Enumerator(this) else Camera1Enumerator(false) +// val frontCamera = cameraIterator.deviceNames +// ?.firstOrNull { cameraIterator.isFrontFacing(it) } +// ?: cameraIterator.deviceNames?.first() +// ?: return true +// val videoCapturer = cameraIterator.createCapturer(frontCamera, null) +// +// // Following instruction here: https://stackoverflow.com/questions/55085726/webrtc-create-peerconnectionfactory-object +// val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) +// +// val videoSource = peerConnectionFactory?.createVideoSource(videoCapturer.isScreencast) +// videoCapturer.initialize(surfaceTextureHelper, this, videoSource!!.capturerObserver) +// videoCapturer.startCapture(1280, 720, 30) +// +// +// val localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource) +// +// // create a local audio track +// val audioSource = peerConnectionFactory?.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) +// val audioTrack = peerConnectionFactory?.createAudioTrack("ARDAMSa0", audioSource) + + pipRenderer.setMirror(true) +// localVideoTrack?.addSink(pipRenderer) + + /* + { + "username": "1586847781:@valere35:matrix.org", + "password": "ZzbqbqfT9O2G3WpCpesdts2lyns=", + "ttl": 86400.0, + "uris": ["turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp"] + } + */ + +// val iceServers = ArrayList().apply { +// listOf("turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp", "turns:turn.matrix.org:443?transport=tcp").forEach { +// add( +// PeerConnection.IceServer.builder(it) +// .setUsername("1586847781:@valere35:matrix.org") +// .setPassword("ZzbqbqfT9O2G3WpCpesdts2lyns=") +// .createIceServer() +// ) +// } +// } +// +// val iceCandidateSource: PublishSubject = PublishSubject.create() +// +// iceCandidateSource +// .buffer(400, TimeUnit.MILLISECONDS) +// .subscribe { +// // omit empty :/ +// if (it.isNotEmpty()) { +// callViewModel.handle(VectorCallViewActions.AddLocalIceCandidate(it)) +// } +// } +// .disposeOnDestroy() +// +// peerConnection = peerConnectionFactory?.createPeerConnection( +// iceServers, +// object : PeerConnectionObserverAdapter() { +// override fun onIceCandidate(p0: IceCandidate?) { +// p0?.let { +// iceCandidateSource.onNext(it) +// } +// } +// +// override fun onAddStream(mediaStream: MediaStream?) { +// runOnUiThread { +// mediaStream?.videoTracks?.firstOrNull()?.let { videoTrack -> +// remoteVideoTrack = videoTrack +// remoteVideoTrack?.setEnabled(true) +// remoteVideoTrack?.addSink(fullscreenRenderer) +// } +// } +// } +// +// override fun onRemoveStream(mediaStream: MediaStream?) { +// remoteVideoTrack = null +// } +// +// override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { +// if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) { +// // TODO prompt something? +// finish() +// } +// } +// } +// ) +// +// val localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? +// localMediaStream?.addTrack(localVideoTrack) +// localMediaStream?.addTrack(audioTrack) +// +// val constraints = MediaConstraints() +// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) +// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")) +// +// peerConnection?.addStream(localMediaStream) +// +// peerConnection?.createOffer(object : SdpObserver { +// override fun onSetFailure(p0: String?) { +// Timber.v("## VOIP onSetFailure $p0") +// } +// +// override fun onSetSuccess() { +// Timber.v("## VOIP onSetSuccess") +// } +// +// override fun onCreateSuccess(sessionDescription: SessionDescription) { +// Timber.v("## VOIP onCreateSuccess $sessionDescription") +// peerConnection?.setLocalDescription(object : SdpObserverAdapter() { +// override fun onSetSuccess() { +// callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) +// } +// }, sessionDescription) +// } +// +// override fun onCreateFailure(p0: String?) { +// Timber.v("## VOIP onCreateFailure $p0") +// } +// }, constraints) + iceCandidateSource + .buffer(400, TimeUnit.MILLISECONDS) + .subscribe { + // omit empty :/ + if (it.isNotEmpty()) { + callViewModel.handle(VectorCallViewActions.AddLocalIceCandidate(it)) + } + } + .disposeOnDestroy() + + peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer) + return false + } + + override fun onDestroy() { + peerConnectionManager.detachRenderers() + peerConnectionManager.listener = this + super.onDestroy() + } + + private fun handleViewEvents(event: VectorCallViewEvents?) { + when (event) { + is VectorCallViewEvents.CallAnswered -> { + val sdp = SessionDescription(SessionDescription.Type.ANSWER, event.content.answer.sdp) + peerConnectionManager.answerReceived("", sdp) +// peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, sdp) + } + } + } + +// @TargetApi(17) +// private fun getDisplayMetrics(): DisplayMetrics? { +// val displayMetrics = DisplayMetrics() +// val windowManager = application.getSystemService(Context.WINDOW_SERVICE) as WindowManager +// windowManager.defaultDisplay.getRealMetrics(displayMetrics) +// return displayMetrics +// } + +// @TargetApi(21) +// private fun startScreenCapture() { +// val mediaProjectionManager: MediaProjectionManager = application.getSystemService( +// Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager +// startActivityForResult( +// mediaProjectionManager.createScreenCaptureIntent(), CAPTURE_PERMISSION_REQUEST_CODE) +// } +// +// override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) { +// if (requestCode != CAPTURE_PERMISSION_REQUEST_CODE) { +// super.onActivityResult(requestCode, resultCode, data) +// } +//// mediaProjectionPermissionResultCode = resultCode; +//// mediaProjectionPermissionResultData = data; +//// startCall(); +// } + + companion object { + + private const val CAPTURE_PERMISSION_REQUEST_CODE = 1 + +// private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints().apply { +// // add all existing audio filters to avoid having echos +// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation2", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googDAEchoCancellation", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl2", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression2", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googAudioMirroring", "false")) +// mandatory.add(MediaConstraints.KeyValuePair("googHighpassFilter", "true")) +// } + + fun newIntent(context: Context, signalingRoomId: String): Intent { + return Intent(context, VectorCallActivity::class.java).apply { + putExtra(MvRx.KEY_ARG, CallArgs(roomId = signalingRoomId)) + } + } + } + + override fun addLocalIceCandidate(candidates: IceCandidate) { + iceCandidateSource.onNext(candidates) + } + + override fun addRemoteVideoTrack(videoTrack: VideoTrack) { + runOnUiThread { + videoTrack.setEnabled(true) + videoTrack.addSink(fullscreenRenderer) + } + } + + override fun addLocalVideoTrack(videoTrack: VideoTrack) { + runOnUiThread { + videoTrack.addSink(pipRenderer) + } + } + + override fun removeRemoteVideoStream(mediaStream: MediaStream) { + } + + override fun onDisconnect() { + } + + override fun sendOffer(sessionDescription: SessionDescription) { + callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt new file mode 100644 index 0000000000..a995e3197e --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorCallViewModel.kt @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import androidx.lifecycle.viewModelScope +import com.airbnb.mvrx.MvRxState +import com.airbnb.mvrx.MvRxViewModelFactory +import com.airbnb.mvrx.ViewModelContext +import com.squareup.inject.assisted.Assisted +import com.squareup.inject.assisted.AssistedInject +import im.vector.matrix.android.api.session.Session +import im.vector.matrix.android.api.session.call.CallsListener +import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent +import im.vector.matrix.android.api.session.room.model.call.CallInviteContent +import im.vector.matrix.android.internal.util.awaitCallback +import im.vector.riotx.core.extensions.exhaustive +import im.vector.riotx.core.platform.VectorViewEvents +import im.vector.riotx.core.platform.VectorViewModel +import im.vector.riotx.core.platform.VectorViewModelAction +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import org.webrtc.IceCandidate +import org.webrtc.SessionDescription +import java.util.UUID + +data class VectorCallViewState( + val callId: String? = null, + val roomId: String = "" +) : MvRxState + +sealed class VectorCallViewActions : VectorViewModelAction { + + data class SendOffer(val sdp: SessionDescription) : VectorCallViewActions() + data class AddLocalIceCandidate(val iceCandidates: List) : VectorCallViewActions() +} + +sealed class VectorCallViewEvents : VectorViewEvents { + + data class CallAnswered(val content: CallAnswerContent) : VectorCallViewEvents() +} + +class VectorCallViewModel @AssistedInject constructor( + @Assisted initialState: VectorCallViewState, + val session: Session +) : VectorViewModel(initialState) { + + private val callServiceListener: CallsListener = object : CallsListener { + override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { + withState { state -> + if (callAnswerContent.callId == state.callId) { + _viewEvents.post(VectorCallViewEvents.CallAnswered(callAnswerContent)) + } + } + } + + override fun onCallInviteReceived(signalingRoomId: String, callInviteContent: CallInviteContent) { + + } + } + + init { + session.callService().addCallListener(callServiceListener) + } + + override fun onCleared() { + session.callService().removeCallListener(callServiceListener) + super.onCleared() + } + + override fun handle(action: VectorCallViewActions) = withState { state -> + when (action) { + is VectorCallViewActions.SendOffer -> { + viewModelScope.launch(Dispatchers.IO) { + awaitCallback { + val callId = state.callId ?: UUID.randomUUID().toString().also { + setState { + copy(callId = it) + } + } + session.callService().sendOfferSdp(callId, state.roomId, action.sdp, it) + } + } + } + is VectorCallViewActions.AddLocalIceCandidate -> { + viewModelScope.launch { + session.callService().sendLocalIceCandidates(state.callId ?: "", state.roomId, action.iceCandidates) + } + } + }.exhaustive + } + + @AssistedInject.Factory + interface Factory { + fun create(initialState: VectorCallViewState): VectorCallViewModel + } + + companion object : MvRxViewModelFactory { + + @JvmStatic + override fun create(viewModelContext: ViewModelContext, state: VectorCallViewState): VectorCallViewModel? { + val callActivity: VectorCallActivity = viewModelContext.activity() + return callActivity.viewModelFactory.create(state) + } + + override fun initialState(viewModelContext: ViewModelContext): VectorCallViewState? { + val args: CallArgs = viewModelContext.args() + return VectorCallViewState(roomId = args.roomId) + } + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/VectorConnectionService.kt b/vector/src/main/java/im/vector/riotx/features/call/VectorConnectionService.kt new file mode 100644 index 0000000000..633f2482cf --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/VectorConnectionService.kt @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import android.os.Build +import android.telecom.Connection +import android.telecom.ConnectionRequest +import android.telecom.ConnectionService +import android.telecom.PhoneAccountHandle +import androidx.annotation.RequiresApi + +/** + * No active calls in other apps + * + *To answer incoming calls when there are no active calls in other apps, follow these steps: + * + * 

+ *     * Your app receives a new incoming call using its usual mechanisms.
+ *          - Use the addNewIncomingCall(PhoneAccountHandle, Bundle) method to inform the telecom subsystem about the new incoming call.
+ *          - The telecom subsystem binds to your app's ConnectionService implementation and requests a new instance of the
+ *            Connection class representing the new incoming call using the onCreateIncomingConnection(PhoneAccountHandle, ConnectionRequest) method.
+ *          - The telecom subsystem informs your app that it should show its incoming call user interface using the onShowIncomingCallUi() method.
+ *          - Your app shows its incoming UI using a notification with an associated full-screen intent. For more information, see onShowIncomingCallUi().
+ *          - Call the setActive() method if the user accepts the incoming call, or setDisconnected(DisconnectCause) specifying REJECTED as
+ *            the parameter followed by a call to the destroy() method if the user rejects the incoming call.
+ *
+ */ +@RequiresApi(Build.VERSION_CODES.M) class VectorConnectionService : ConnectionService() { + + /** + * The telecom subsystem calls this method in response to your app calling placeCall(Uri, Bundle) to create a new outgoing call + */ + override fun onCreateOutgoingConnection(connectionManagerPhoneAccount: PhoneAccountHandle?, request: ConnectionRequest?): Connection? { + val callId = request?.address?.encodedQuery ?: return null + val roomId = request.extras.getString("MX_CALL_ROOM_ID") ?: return null + return CallConnection(applicationContext, roomId, callId) + } +} diff --git a/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt new file mode 100644 index 0000000000..7833ac1787 --- /dev/null +++ b/vector/src/main/java/im/vector/riotx/features/call/WebRtcPeerConnectionManager.kt @@ -0,0 +1,372 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.riotx.features.call + +import android.content.ComponentName +import android.content.Context +import android.graphics.drawable.Icon +import android.os.Build +import android.os.Bundle +import android.telecom.PhoneAccount +import android.telecom.PhoneAccountHandle +import android.telecom.TelecomManager +import androidx.core.content.ContextCompat +import im.vector.matrix.android.api.session.call.CallsListener +import im.vector.matrix.android.api.session.call.EglUtils +import im.vector.matrix.android.api.session.room.model.call.CallAnswerContent +import im.vector.matrix.android.api.session.room.model.call.CallInviteContent +import im.vector.riotx.BuildConfig +import im.vector.riotx.R +import org.webrtc.AudioSource +import org.webrtc.AudioTrack +import org.webrtc.DefaultVideoDecoderFactory +import org.webrtc.DefaultVideoEncoderFactory +import org.webrtc.IceCandidate +import org.webrtc.MediaConstraints +import org.webrtc.MediaStream +import org.webrtc.PeerConnection +import org.webrtc.PeerConnectionFactory +import org.webrtc.SdpObserver +import org.webrtc.SessionDescription +import org.webrtc.SurfaceTextureHelper +import org.webrtc.SurfaceViewRenderer +import org.webrtc.VideoCapturer +import org.webrtc.VideoSource +import org.webrtc.VideoTrack +import timber.log.Timber +import java.lang.ref.WeakReference +import java.util.concurrent.Executors +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Manage peerConnectionFactory & Peer connections outside of activity lifecycle to resist configuration changes + * Use app context + */ +@Singleton +class WebRtcPeerConnectionManager @Inject constructor( + private val context: Context +) : CallsListener { + + interface Listener { + fun addLocalIceCandidate(candidates: IceCandidate) + fun addRemoteVideoTrack(videoTrack: VideoTrack) + fun addLocalVideoTrack(videoTrack: VideoTrack) + fun removeRemoteVideoStream(mediaStream: MediaStream) + fun onDisconnect() + fun sendOffer(sessionDescription: SessionDescription) + } + + var phoneAccountHandle: PhoneAccountHandle? = null + + init { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + val componentName = ComponentName(BuildConfig.APPLICATION_ID, VectorConnectionService::class.java.name) + val appName = context.getString(R.string.app_name) + phoneAccountHandle = PhoneAccountHandle(componentName, appName) + val phoneAccount = PhoneAccount.Builder(phoneAccountHandle, appName) + .setIcon(Icon.createWithResource(context, R.drawable.riotx_logo)) + .build() + ContextCompat.getSystemService(context, TelecomManager::class.java) + ?.registerPhoneAccount(phoneAccount) + } else { + // ignore? + } + } + + var listener: Listener? = null + + // *Comments copied from webrtc demo app* + // Executor thread is started once and is used for all + // peer connection API calls to ensure new peer connection factory is + // created on the same thread as previously destroyed factory. + private val executor = Executors.newSingleThreadExecutor(); + + private val rootEglBase by lazy { EglUtils.rootEglBase } + + private var peerConnectionFactory: PeerConnectionFactory? = null + + private var peerConnection: PeerConnection? = null + + private var remoteVideoTrack: VideoTrack? = null + private var localVideoTrack: VideoTrack? = null + + private var videoSource: VideoSource? = null + private var audioSource: AudioSource? = null + private var audioTrack: AudioTrack? = null + + private var videoCapturer: VideoCapturer? = null + + var localSurfaceRenderer: WeakReference? = null + var remoteSurfaceRenderer: WeakReference? = null + + fun createPeerConnectionFactory() { + executor.execute { + if (peerConnectionFactory == null) { + Timber.v("## VOIP createPeerConnectionFactory") + val eglBaseContext = rootEglBase?.eglBaseContext ?: return@execute Unit.also { + Timber.e("## VOIP No EGL BASE") + } + + Timber.v("## VOIP PeerConnectionFactory.initialize") + PeerConnectionFactory.initialize(PeerConnectionFactory + .InitializationOptions.builder(context.applicationContext) + .createInitializationOptions() + ) + + val options = PeerConnectionFactory.Options() + val defaultVideoEncoderFactory = DefaultVideoEncoderFactory( + eglBaseContext, + /* enableIntelVp8Encoder */ + true, + /* enableH264HighProfile */ + true) + val defaultVideoDecoderFactory = DefaultVideoDecoderFactory(eglBaseContext) + + + Timber.v("## VOIP PeerConnectionFactory.createPeerConnectionFactory ...") + peerConnectionFactory = PeerConnectionFactory.builder() + .setOptions(options) + .setVideoEncoderFactory(defaultVideoEncoderFactory) + .setVideoDecoderFactory(defaultVideoDecoderFactory) + .createPeerConnectionFactory() + } + } + } + + fun createPeerConnection(videoCapturer: VideoCapturer, iceServers: List) { + executor.execute { + Timber.v("## VOIP PeerConnectionFactory.createPeerConnection ${peerConnectionFactory}...") + // Following instruction here: https://stackoverflow.com/questions/55085726/webrtc-create-peerconnectionfactory-object + val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) + + videoSource = peerConnectionFactory?.createVideoSource(videoCapturer.isScreencast) + Timber.v("## VOIP Local video source created") + videoCapturer.initialize(surfaceTextureHelper, context.applicationContext, videoSource!!.capturerObserver) + videoCapturer.startCapture(1280, 720, 30) + + localVideoTrack = peerConnectionFactory?.createVideoTrack("ARDAMSv0", videoSource)?.also { + Timber.v("## VOIP Local video track created") + listener?.addLocalVideoTrack(it) +// localSurfaceRenderer?.get()?.let { surface -> +//// it.addSink(surface) +//// } + } + + // create a local audio track + Timber.v("## VOIP create local audio track") + audioSource = peerConnectionFactory?.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) + audioTrack = peerConnectionFactory?.createAudioTrack(AUDIO_TRACK_ID, audioSource) + +// pipRenderer.setMirror(true) +// localVideoTrack?.addSink(pipRenderer) +// + +// val iceCandidateSource: PublishSubject = PublishSubject.create() +// +// iceCandidateSource +// .buffer(400, TimeUnit.MILLISECONDS) +// .subscribe { +// // omit empty :/ +// if (it.isNotEmpty()) { +// listener.addLocalIceCandidate() +// callViewModel.handle(VectorCallViewActions.AddLocalIceCandidate(it)) +// } +// } +// .disposeOnDestroy() + + Timber.v("## VOIP creating peer connection... ") + peerConnection = peerConnectionFactory?.createPeerConnection( + iceServers, + object : PeerConnectionObserverAdapter() { + override fun onIceCandidate(p0: IceCandidate?) { + Timber.v("## VOIP onIceCandidate local $p0") + p0?.let { + // iceCandidateSource.onNext(it) + listener?.addLocalIceCandidate(it) + } + } + + override fun onAddStream(mediaStream: MediaStream?) { + Timber.v("## VOIP onAddStream remote $mediaStream") + mediaStream?.videoTracks?.firstOrNull()?.let { + listener?.addRemoteVideoTrack(it) + remoteVideoTrack = it +// remoteSurfaceRenderer?.get()?.let { surface -> +// it.setEnabled(true) +// it.addSink(surface) +// } + } +// runOnUiThread { +// mediaStream?.videoTracks?.firstOrNull()?.let { videoTrack -> +// remoteVideoTrack = videoTrack +// remoteVideoTrack?.setEnabled(true) +// remoteVideoTrack?.addSink(fullscreenRenderer) +// } +// } + } + + override fun onRemoveStream(mediaStream: MediaStream?) { + mediaStream?.let { + listener?.removeRemoteVideoStream(it) + } + remoteSurfaceRenderer?.get()?.let { + remoteVideoTrack?.removeSink(it) + } + remoteVideoTrack = null + } + + override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) { + Timber.v("## VOIP onIceConnectionChange $p0") + if (p0 == PeerConnection.IceConnectionState.DISCONNECTED) { + listener?.onDisconnect() + } + } + } + ) + + val localMediaStream = peerConnectionFactory?.createLocalMediaStream("ARDAMS") // magic value? + localMediaStream?.addTrack(localVideoTrack) + localMediaStream?.addTrack(audioTrack) + +// val constraints = MediaConstraints() +// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) +// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")) + + Timber.v("## VOIP add local stream to peer connection") + peerConnection?.addStream(localMediaStream) + } + } + + fun answerReceived(callId: String, answerSdp: SessionDescription) { + executor.execute { + Timber.v("## answerReceived $callId") + peerConnection?.setRemoteDescription(object : SdpObserverAdapter() {}, answerSdp) + } + } + + fun startCall() { + executor.execute { + val constraints = MediaConstraints() + constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) + constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")) + + Timber.v("## VOIP creating offer...") + peerConnection?.createOffer(object : SdpObserver { + override fun onSetFailure(p0: String?) { + Timber.v("## VOIP onSetFailure $p0") + } + + override fun onSetSuccess() { + Timber.v("## VOIP onSetSuccess") + } + + override fun onCreateSuccess(sessionDescription: SessionDescription) { + Timber.v("## VOIP onCreateSuccess $sessionDescription") + peerConnection?.setLocalDescription(object : SdpObserverAdapter() { + override fun onSetSuccess() { + listener?.sendOffer(sessionDescription) + //callViewModel.handle(VectorCallViewActions.SendOffer(sessionDescription)) + } + }, sessionDescription) + } + + override fun onCreateFailure(p0: String?) { + Timber.v("## VOIP onCreateFailure $p0") + } + }, constraints) + } + } + + fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer, remoteViewRenderer: SurfaceViewRenderer) { + localVideoTrack?.addSink(localViewRenderer) + remoteVideoTrack?.let { + it.setEnabled(true) + it.addSink(remoteViewRenderer) + } + localSurfaceRenderer = WeakReference(localViewRenderer) + remoteSurfaceRenderer = WeakReference(remoteViewRenderer) + } + + fun detachRenderers() { + localSurfaceRenderer?.get()?.let { + localVideoTrack?.removeSink(it) + } + remoteSurfaceRenderer?.get()?.let { + remoteVideoTrack?.removeSink(it) + } + localSurfaceRenderer = null + remoteSurfaceRenderer = null + } + + fun close() { + executor.execute { + peerConnectionFactory?.stopAecDump() + peerConnectionFactory = null + audioSource?.dispose() + videoSource?.dispose() + peerConnection?.dispose() + peerConnection = null + videoCapturer?.dispose() + } + } + + companion object { + + private const val AUDIO_TRACK_ID = "ARDAMSa0" + + private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints().apply { + // add all existing audio filters to avoid having echos + mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation", "true")) + mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation2", "true")) + mandatory.add(MediaConstraints.KeyValuePair("googDAEchoCancellation", "true")) + + mandatory.add(MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true")) + + mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl", "true")) + mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl2", "true")) + + mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression", "true")) + mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression2", "true")) + + mandatory.add(MediaConstraints.KeyValuePair("googAudioMirroring", "false")) + mandatory.add(MediaConstraints.KeyValuePair("googHighpassFilter", "true")) + } + } + + override fun onCallInviteReceived(signalingRoomId: String, callInviteContent: CallInviteContent) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + ContextCompat.getSystemService(context, TelecomManager::class.java)?.let { telecomManager -> + phoneAccountHandle?.let { phoneAccountHandle -> + telecomManager.addNewIncomingCall( + phoneAccountHandle, + Bundle().apply { + putString("MX_CALL_ROOM_ID", signalingRoomId) + putString("MX_CALL_CALL_ID", callInviteContent.callId) + } + ) + } + + } + } + } + + override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { + } +} + + diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt index fba4f9e79e..896b29009e 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailAction.kt @@ -68,6 +68,7 @@ sealed class RoomDetailAction : VectorViewModelAction { object ClearSendQueue : RoomDetailAction() object ResendAll : RoomDetailAction() + object StartCall : RoomDetailAction() data class AcceptVerificationRequest(val transactionId: String, val otherUserId: String) : RoomDetailAction() data class DeclineVerificationRequest(val transactionId: String, val otherUserId: String) : RoomDetailAction() diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt index bf386d15a2..2b743ed940 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailFragment.kt @@ -127,6 +127,7 @@ import im.vector.riotx.features.attachments.ContactAttachment import im.vector.riotx.features.attachments.preview.AttachmentsPreviewActivity import im.vector.riotx.features.attachments.preview.AttachmentsPreviewArgs import im.vector.riotx.features.attachments.toGroupedContentAttachmentData +import im.vector.riotx.features.call.VectorCallActivity import im.vector.riotx.features.command.Command import im.vector.riotx.features.crypto.keysbackup.restore.KeysBackupRestoreActivity import im.vector.riotx.features.crypto.util.toImageRes @@ -479,6 +480,17 @@ class RoomDetailFragment @Inject constructor( } else -> super.onOptionsItemSelected(item) } + if (item.itemId == R.id.resend_all) { + roomDetailViewModel.handle(RoomDetailAction.ResendAll) + return true + } + if (item.itemId == R.id.voip_call) { + VectorCallActivity.newIntent(requireContext(), roomDetailArgs.roomId).let { + startActivity(it) + } + return true + } + return super.onOptionsItemSelected(item) } private fun displayDisabledIntegrationDialog() { diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt index 1785151af5..c25bd6c8da 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/RoomDetailViewModel.kt @@ -66,6 +66,7 @@ import im.vector.riotx.core.platform.VectorViewModel import im.vector.riotx.core.resources.StringProvider import im.vector.riotx.core.resources.UserPreferencesProvider import im.vector.riotx.core.utils.subscribeLogError +import im.vector.riotx.features.call.VectorCallActivity import im.vector.riotx.features.command.CommandParser import im.vector.riotx.features.command.ParsedCommand import im.vector.riotx.features.crypto.verification.SupportedVerificationMethodsProvider @@ -372,6 +373,7 @@ class RoomDetailViewModel @AssistedInject constructor( R.id.resend_all -> timeline.failedToDeliverEventCount() > 0 R.id.clear_all -> timeline.failedToDeliverEventCount() > 0 R.id.open_matrix_apps -> true + R.id.voip_call -> room.roomSummary()?.isDirect == true && room.roomSummary()?.joinedMembersCount == 2 else -> false } diff --git a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt index 33079a8f33..c1ec2d1cac 100644 --- a/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt +++ b/vector/src/main/java/im/vector/riotx/features/home/room/detail/timeline/format/NoticeEventFormatter.kt @@ -239,7 +239,7 @@ class NoticeEventFormatter @Inject constructor(private val sessionHolder: Active return when (type) { EventType.CALL_INVITE -> { val content = event.getClearContent().toModel() ?: return null - val isVideoCall = content.offer.sdp == CallInviteContent.Offer.SDP_VIDEO + val isVideoCall = content.offer?.sdp == CallInviteContent.Offer.SDP_VIDEO return if (isVideoCall) { if (event.isSentByCurrentUser()) { sp.getString(R.string.notice_placed_video_call_by_you) diff --git a/vector/src/main/res/drawable/ic_phone.xml b/vector/src/main/res/drawable/ic_phone.xml new file mode 100644 index 0000000000..430c438577 --- /dev/null +++ b/vector/src/main/res/drawable/ic_phone.xml @@ -0,0 +1,14 @@ + + + diff --git a/vector/src/main/res/layout/activity_call.xml b/vector/src/main/res/layout/activity_call.xml new file mode 100644 index 0000000000..f1a2a3075f --- /dev/null +++ b/vector/src/main/res/layout/activity_call.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/vector/src/main/res/layout/fragment_call.xml b/vector/src/main/res/layout/fragment_call.xml new file mode 100644 index 0000000000..2ab342d68a --- /dev/null +++ b/vector/src/main/res/layout/fragment_call.xml @@ -0,0 +1,63 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/vector/src/main/res/menu/menu_timeline.xml b/vector/src/main/res/menu/menu_timeline.xml index a4adb203f2..8e271a0285 100644 --- a/vector/src/main/res/menu/menu_timeline.xml +++ b/vector/src/main/res/menu/menu_timeline.xml @@ -8,6 +8,14 @@ android:title="@string/room_add_matrix_apps" app:showAsAction="never" /> + +