Merge pull request #160 from vector-im/conferencing

Add conferencing support
This commit is contained in:
David Baker 2015-09-18 10:03:02 +01:00
commit 81db1b2360
13 changed files with 423 additions and 62 deletions

52
docs/conferencing.md Normal file
View file

@ -0,0 +1,52 @@
# VoIP Conferencing
This is a draft proposal for a naive voice/video conferencing implementation for
Matrix clients. There are many possible conferencing architectures possible for
Matrix (Multipoint Conferencing Unit (MCU); Stream Forwarding Unit (SFU); Peer-
to-Peer mesh (P2P), etc; events shared in the group room; events shared 1:1;
possibly even out-of-band signalling).
This is a starting point for a naive MCU implementation which could provide one
possible Matrix-wide solution in future, which retains backwards compatibility
with standard 1:1 calling.
* A client chooses to initiate a conference for a given room by starting a
voice or video call with a 'conference focus' user. This is a virtual user
(typically Application Service) which implements a conferencing bridge. It
isn't defined how the client discovers or selects this user.
* The conference focus user MUST join the room in which the client has
initiated the conference - this may require the client to invite the
conference focus user to the room, depending on the room's `join_rules`. The
conference focus user needs to be in the room to let the bridge eject users
from the conference who have left the room in which it was initiated, and aid
discovery of the conference by other users in the room. The bridge
identifies the room to join based on the user ID by which it was invited.
The format of this identifier is implementation dependent for now.
* If a client leaves the group chat room, they MUST be ejected from the
conference. If a client leaves the 1:1 room with the conference focus user,
they SHOULD be ejected from the conference.
* For now, rooms can contain multiple conference focus users - it's left to
user or client implementation to select which to converge on. In future this
could be mediated using a state event (e.g. `im.vector.call.mcu`), but we
can't do that right now as by default normal users can't set arbitrary state
events on a room.
* To participate in the conference, other clients initiates a standard 1:1
voice or video call to the conference focus user.
* For best UX, clients SHOULD show the ongoing voice/video call in the UI
context of the group room rather than 1:1 with the focus user. If a client
recognises a conference user present in the room, it MAY chose to highlight
this in the UI (e.g. with a "conference ongoing" notification, to aid
discovery). Clients MAY hide the 1:1 room with the focus user (although in
future this room could be used for floor control or other direct
communication with the conference focus)
* When all users have left the conference, the 'conference focus' user SHOULD
leave the room.
* If a conference focus user joins a room but does not receive a 1:1 voice or
video call, it SHOULD time out after a period of time and leave the room.

View file

@ -218,3 +218,12 @@ limitations under the License.
background-color: blue;
height: 5px;
}
.mx_RoomView_ongoingConfCallNotification {
width: 100%;
text-align: center;
background-color: #ff0064;
color: #fff;
font-weight: bold;
padding: 6px;
}

View file

@ -75,6 +75,22 @@ limitations under the License.
opacity: 0.8;
}
.mx_Login_create:link {
color: #4a4a4a;
}
.mx_Login_links {
display: block;
text-align: center;
width: 100%;
font-size: 14px;
opacity: 0.8;
}
.mx_Login_links a:link {
color: #4a4a4a;
}
.mx_Login_loader {
position: absolute;
left: 50%;
@ -85,12 +101,10 @@ limitations under the License.
color: #ff2020;
font-weight: bold;
text-align: center;
/*
height: 24px;
*/
margin-top: 12px;
margin-bottom: 12px;
}
.mx_Login_create:link {
color: #4a4a4a;
}

View file

@ -18,7 +18,7 @@ limitations under the License.
var React = require('react');
var ComponentBroker = require('../../../../src/ComponentBroker');
var CallView = ComponentBroker.get('molecules/voip/CallView');
var RoomDropTarget = ComponentBroker.get('molecules/RoomDropTarget');
var RoomListController = require("../../../../src/controllers/organisms/RoomList");
@ -28,8 +28,14 @@ module.exports = React.createClass({
mixins: [RoomListController],
render: function() {
var callElement;
if (this.state.show_call_element) {
callElement = <CallView className="mx_MatrixChat_callView"/>
}
return (
<div className="mx_RoomList">
{callElement}
<h2 className="mx_RoomList_favourites_label">Favourites</h2>
<RoomDropTarget text="Drop here to favourite"/>

View file

@ -176,6 +176,15 @@ module.exports = React.createClass({
roomEdit = <Loader/>;
}
var conferenceCallNotification = null;
if (this.state.displayConfCallNotification) {
conferenceCallNotification = (
<div className="mx_RoomView_ongoingConfCallNotification" onClick={this.onConferenceNotificationClick}>
Ongoing conference call
</div>
);
}
var fileDropTarget = null;
if (this.state.draggingFile) {
fileDropTarget = <div className="mx_RoomView_fileDropTarget">
@ -192,6 +201,7 @@ module.exports = React.createClass({
onSettingsClick={this.onSettingsClick} onSaveClick={this.onSaveClick} onCancelClick={this.onCancelClick} />
<div className="mx_RoomView_auxPanel">
<CallView room={this.state.room}/>
{ conferenceCallNotification }
{ roomEdit }
</div>
<div ref="messageWrapper" className="mx_RoomView_messagePanel" onScroll={ this.onMessageListScroll }>

View file

@ -30,7 +30,7 @@ var RoomDirectory = ComponentBroker.get('organisms/RoomDirectory');
var MatrixToolbar = ComponentBroker.get('molecules/MatrixToolbar');
var Notifier = ComponentBroker.get('organisms/Notifier');
var MatrixChatController = require("../../../../src/controllers/pages/MatrixChat");
var MatrixChatController = require('../../../../src/controllers/pages/MatrixChat');
// should be atomised
var Loader = require("react-loader");
@ -75,6 +75,7 @@ module.exports = React.createClass({
break;
}
// TODO: Fix duplication here and do conditionals like we do above
if (Notifier.supportsDesktopNotifications() && !Notifier.isEnabled() && !Notifier.isToolbarHidden()) {
return (
<div className="mx_MatrixChat_wrapper">

View file

@ -165,6 +165,13 @@ module.exports = React.createClass({
{this.state.errorText}
</div>
<a className="mx_Login_create" onClick={this.showRegister} href="#">Create a new account</a>
<br/>
<div className="mx_Login_links">
<a href="https://medium.com/@Vector">blog</a>&nbsp;&nbsp;&middot;&nbsp;&nbsp;
<a href="https://twitter.com/@VectorCo">twitter</a>&nbsp;&nbsp;&middot;&nbsp;&nbsp;
<a href="https://github.com/vector-im/vector-web">github</a>&nbsp;&nbsp;&middot;&nbsp;&nbsp;
<a href="https://matrix.org">powered by Matrix</a>
</div>
</div>
);
},

View file

@ -57,6 +57,8 @@ var MatrixClientPeg = require("./MatrixClientPeg");
var Modal = require("./Modal");
var ComponentBroker = require('./ComponentBroker');
var ErrorDialog = ComponentBroker.get("organisms/ErrorDialog");
var ConferenceCall = require("./ConferenceHandler").ConferenceCall;
var ConferenceHandler = require("./ConferenceHandler");
var Matrix = require("matrix-js-sdk");
var dis = require("./dispatcher");
@ -105,7 +107,7 @@ function _setCallListeners(call) {
play("ringbackAudio");
}
else if (newState === "ended" && oldState === "connected") {
_setCallState(call, call.roomId, "ended");
_setCallState(undefined, call.roomId, "ended");
pause("ringbackAudio");
play("callendAudio");
}
@ -153,7 +155,11 @@ function _setCallState(call, roomId, status) {
dis.register(function(payload) {
switch (payload.action) {
case 'place_call':
if (calls[payload.room_id]) {
if (module.exports.getAnyActiveCall()) {
Modal.createDialog(ErrorDialog, {
title: "Existing Call",
description: "You are already in a call."
});
return; // don't allow >1 call to be placed.
}
var room = MatrixClientPeg.get().getRoom(payload.room_id);
@ -161,40 +167,52 @@ dis.register(function(payload) {
console.error("Room %s does not exist.", payload.room_id);
return;
}
var members = room.getJoinedMembers();
if (members.length !== 2) {
var text = members.length === 1 ? "yourself." : "more than 2 people.";
Modal.createDialog(ErrorDialog, {
description: "You cannot place a call with " + text
});
console.error(
"Fail: There are %s joined members in this room, not 2.",
room.getJoinedMembers().length
);
return;
}
console.log("Place %s call in %s", payload.type, payload.room_id);
var call = Matrix.createNewMatrixCall(
MatrixClientPeg.get(), payload.room_id
);
_setCallListeners(call);
_setCallState(call, call.roomId, "ringback");
if (payload.type === 'voice') {
call.placeVoiceCall();
}
else if (payload.type === 'video') {
call.placeVideoCall(
payload.remote_element,
payload.local_element
);
}
else {
console.error("Unknown call type: %s", payload.type);
function placeCall(newCall) {
_setCallListeners(newCall);
_setCallState(newCall, newCall.roomId, "ringback");
if (payload.type === 'voice') {
newCall.placeVoiceCall();
}
else if (payload.type === 'video') {
newCall.placeVideoCall(
payload.remote_element,
payload.local_element
);
}
else {
console.error("Unknown conf call type: %s", payload.type);
}
}
var members = room.getJoinedMembers();
if (members.length <= 1) {
Modal.createDialog(ErrorDialog, {
description: "You cannot place a call with yourself."
});
return;
}
else if (members.length === 2) {
console.log("Place %s call in %s", payload.type, payload.room_id);
var call = Matrix.createNewMatrixCall(
MatrixClientPeg.get(), payload.room_id
);
placeCall(call);
}
else { // > 2
console.log("Place conference call in %s", payload.room_id);
var confCall = new ConferenceCall(
MatrixClientPeg.get(), payload.room_id
);
confCall.setup().done(function(call) {
placeCall(call);
}, function(err) {
console.error("Failed to setup conference call: %s", err);
});
}
break;
case 'incoming_call':
if (calls[payload.call.roomId]) {
if (module.exports.getAnyActiveCall()) {
payload.call.hangup("busy");
return; // don't allow >1 call to be received, hangup newer one.
}
@ -224,7 +242,40 @@ dis.register(function(payload) {
});
module.exports = {
getCallForRoom: function(roomId) {
return (
module.exports.getCall(roomId) ||
module.exports.getConferenceCall(roomId)
);
},
getCall: function(roomId) {
return calls[roomId] || null;
},
getConferenceCall: function(roomId) {
// search for a conference 1:1 call for this group chat room ID
var activeCall = module.exports.getAnyActiveCall();
if (activeCall && activeCall.confUserId) {
var thisRoomConfUserId = ConferenceHandler.getConferenceUserIdForRoom(
roomId
);
if (thisRoomConfUserId === activeCall.confUserId) {
return activeCall;
}
}
return null;
},
getAnyActiveCall: function() {
var roomsWithCalls = Object.keys(calls);
for (var i = 0; i < roomsWithCalls.length; i++) {
if (calls[roomsWithCalls[i]] &&
calls[roomsWithCalls[i]].call_state !== "ended") {
return calls[roomsWithCalls[i]];
}
}
return null;
}
};

94
src/ConferenceHandler.js Normal file
View file

@ -0,0 +1,94 @@
"use strict";
var q = require("q");
var Matrix = require("matrix-js-sdk");
var Room = Matrix.Room;
// FIXME: This currently forces Vector to try to hit the matrix.org AS for conferencing.
// This is bad because it prevents people running their own ASes from being used.
// This isn't permanent and will be customisable in the future: see the proposal
// at docs/conferencing.md for more info.
var USER_PREFIX = "fs_";
var DOMAIN = "matrix.org";
function ConferenceCall(matrixClient, groupChatRoomId) {
this.client = matrixClient;
this.groupRoomId = groupChatRoomId;
this.confUserId = module.exports.getConferenceUserIdForRoom(this.groupRoomId);
}
ConferenceCall.prototype.setup = function() {
var self = this;
return this._joinConferenceUser().then(function() {
return self._getConferenceUserRoom();
}).then(function(room) {
// return a call for *this* room to be placed. We also tack on
// confUserId to speed up lookups (else we'd need to loop every room
// looking for a 1:1 room with this conf user ID!)
var call = Matrix.createNewMatrixCall(self.client, room.roomId);
call.confUserId = self.confUserId;
return call;
});
};
ConferenceCall.prototype._joinConferenceUser = function() {
// Make sure the conference user is in the group chat room
var groupRoom = this.client.getRoom(this.groupRoomId);
if (!groupRoom) {
return q.reject("Bad group room ID");
}
var member = groupRoom.getMember(this.confUserId);
if (member && member.membership === "join") {
return q();
}
return this.client.invite(this.groupRoomId, this.confUserId);
};
ConferenceCall.prototype._getConferenceUserRoom = function() {
// Use an existing 1:1 with the conference user; else make one
var rooms = this.client.getRooms();
var confRoom = null;
for (var i = 0; i < rooms.length; i++) {
var confUser = rooms[i].getMember(this.confUserId);
if (confUser && confUser.membership === "join" &&
rooms[i].getJoinedMembers().length === 2) {
confRoom = rooms[i];
break;
}
}
if (confRoom) {
return q(confRoom);
}
return this.client.createRoom({
preset: "private_chat",
invite: [this.confUserId]
}).then(function(res) {
return new Room(res.room_id);
});
};
/**
* Check if this room member is in fact a conference bot.
* @param {RoomMember} The room member to check
* @return {boolean} True if it is a conference bot.
*/
module.exports.isConferenceUser = function(roomMember) {
if (roomMember.userId.indexOf("@" + USER_PREFIX) !== 0) {
return false;
}
var base64part = roomMember.userId.split(":")[0].substring(1 + USER_PREFIX.length);
if (base64part) {
var decoded = new Buffer(base64part, "base64").toString();
// ! $STUFF : $STUFF
return /^!.+:.+/.test(decoded);
}
return false;
};
module.exports.getConferenceUserIdForRoom = function(roomId) {
// abuse browserify's core node Buffer support (strip padding ='s)
var base64RoomId = new Buffer(roomId).toString("base64").replace(/=/g, "");
return "@" + USER_PREFIX + base64RoomId + ":" + DOMAIN;
};
module.exports.ConferenceCall = ConferenceCall;

View file

@ -19,6 +19,9 @@ limitations under the License.
/*
* State vars:
* this.state.call_state = the UI state of the call (see CallHandler)
*
* Props:
* room (JS SDK Room)
*/
var React = require('react');
@ -44,7 +47,7 @@ module.exports = {
componentDidMount: function() {
this.dispatcherRef = dis.register(this.onAction);
if (this.props.room) {
var call = CallHandler.getCall(this.props.room.roomId);
var call = CallHandler.getCallForRoom(this.props.room.roomId);
var callState = call ? call.call_state : "ended";
this.setState({
call_state: callState
@ -57,15 +60,12 @@ module.exports = {
},
onAction: function(payload) {
// if we were given a room_id to track, don't handle anything else.
if (payload.room_id && this.props.room &&
this.props.room.roomId !== payload.room_id) {
// don't filter out payloads for room IDs other than props.room because
// we may be interested in the conf 1:1 room
if (payload.action !== 'call_state' || !payload.room_id) {
return;
}
if (payload.action !== 'call_state') {
return;
}
var call = CallHandler.getCall(payload.room_id);
var call = CallHandler.getCallForRoom(payload.room_id);
var callState = call ? call.call_state : "ended";
this.setState({
call_state: callState
@ -87,9 +87,13 @@ module.exports = {
});
},
onHangupClick: function() {
var call = CallHandler.getCallForRoom(this.props.room.roomId);
if (!call) { return; }
dis.dispatch({
action: 'hangup',
room_id: this.props.room.roomId
// hangup the call for this room, which may not be the room in props
// (e.g. conferences which will hangup the 1:1 room instead)
room_id: call.roomId
});
}
};

View file

@ -17,6 +17,7 @@ limitations under the License.
'use strict';
var dis = require("../../../dispatcher");
var CallHandler = require("../../../CallHandler");
var MatrixClientPeg = require("../../../MatrixClientPeg");
/*
* State vars:
@ -24,14 +25,30 @@ var CallHandler = require("../../../CallHandler");
*
* Props:
* this.props.room = Room (JS SDK)
*
* Internal state:
* this._trackedRoom = (either from props.room or programatically set)
*/
module.exports = {
componentDidMount: function() {
this.dispatcherRef = dis.register(this.onAction);
this._trackedRoom = null;
if (this.props.room) {
this.showCall(this.props.room.roomId);
this._trackedRoom = this.props.room;
this.showCall(this._trackedRoom.roomId);
}
else {
var call = CallHandler.getAnyActiveCall();
if (call) {
console.log(
"Global CallView is now tracking active call in room %s",
call.roomId
);
this._trackedRoom = MatrixClientPeg.get().getRoom(call.roomId);
this.showCall(call.roomId);
}
}
},
@ -40,26 +57,27 @@ module.exports = {
},
onAction: function(payload) {
// if we were given a room_id to track, don't handle anything else.
if (payload.room_id && this.props.room &&
this.props.room.roomId !== payload.room_id) {
return;
}
if (payload.action !== 'call_state') {
// don't filter out payloads for room IDs other than props.room because
// we may be interested in the conf 1:1 room
if (payload.action !== 'call_state' || !payload.room_id) {
return;
}
this.showCall(payload.room_id);
},
showCall: function(roomId) {
var call = CallHandler.getCall(roomId);
var call = CallHandler.getCallForRoom(roomId);
if (call) {
call.setLocalVideoElement(this.getVideoView().getLocalVideoElement());
// N.B. the remote video element is used for playback for audio for voice calls
call.setRemoteVideoElement(this.getVideoView().getRemoteVideoElement());
}
if (call && call.type === "video" && call.state !== 'ended') {
this.getVideoView().getLocalVideoElement().style.display = "initial";
// if this call is a conf call, don't display local video as the
// conference will have us in it
this.getVideoView().getLocalVideoElement().style.display = (
call.confUserId ? "none" : "initial"
);
this.getVideoView().getRemoteVideoElement().style.display = "initial";
}
else {

View file

@ -19,11 +19,16 @@ limitations under the License.
var React = require("react");
var MatrixClientPeg = require("../../MatrixClientPeg");
var RoomListSorter = require("../../RoomListSorter");
var dis = require("../../dispatcher");
var ComponentBroker = require('../../ComponentBroker');
var ConferenceHandler = require("../../ConferenceHandler");
var CallHandler = require("../../CallHandler");
var RoomTile = ComponentBroker.get("molecules/RoomTile");
var HIDE_CONFERENCE_CHANS = true;
module.exports = {
componentWillMount: function() {
var cli = MatrixClientPeg.get();
@ -38,7 +43,22 @@ module.exports = {
});
},
componentDidMount: function() {
this.dispatcherRef = dis.register(this.onAction);
},
onAction: function(payload) {
switch (payload.action) {
// listen for call state changes to prod the render method, which
// may hide the global CallView if the call it is tracking is dead
case 'call_state':
this._recheckCallElement(this.props.selectedRoom);
break;
}
},
componentWillUnmount: function() {
dis.unregister(this.dispatcherRef);
if (MatrixClientPeg.get()) {
MatrixClientPeg.get().removeListener("Room", this.onRoom);
MatrixClientPeg.get().removeListener("Room.timeline", this.onRoomTimeline);
@ -48,6 +68,7 @@ module.exports = {
componentWillReceiveProps: function(newProps) {
this.state.activityMap[newProps.selectedRoom] = undefined;
this._recheckCallElement(newProps.selectedRoom);
this.setState({
activityMap: this.state.activityMap
});
@ -96,12 +117,41 @@ module.exports = {
getRoomList: function() {
return RoomListSorter.mostRecentActivityFirst(
MatrixClientPeg.get().getRooms().filter(function(room) {
var member = room.getMember(MatrixClientPeg.get().credentials.userId);
return member && (member.membership == "join" || member.membership == "invite");
var me = room.getMember(MatrixClientPeg.get().credentials.userId);
var shouldShowRoom = (
me && (me.membership == "join" || me.membership == "invite")
);
// hiding conf rooms only ever toggles shouldShowRoom to false
if (shouldShowRoom && HIDE_CONFERENCE_CHANS) {
// we want to hide the 1:1 conf<->user room and not the group chat
var joinedMembers = room.getJoinedMembers();
if (joinedMembers.length === 2) {
var otherMember = joinedMembers.filter(function(m) {
return m.userId !== me.userId
})[0];
if (ConferenceHandler.isConferenceUser(otherMember)) {
// console.log("Hiding conference 1:1 room %s", room.roomId);
shouldShowRoom = false;
}
}
}
return shouldShowRoom;
})
);
},
_recheckCallElement: function(selectedRoomId) {
// if we aren't viewing a room with an ongoing call, but there is an
// active call, show the call element - we need to do this to make
// audio/video not crap out
var activeCall = CallHandler.getAnyActiveCall();
var callForRoom = CallHandler.getCallForRoom(selectedRoomId);
var showCall = (activeCall && !callForRoom);
this.setState({
show_call_element: showCall
});
},
makeRoomTiles: function() {
var self = this;
return this.state.roomList.map(function(room) {
@ -116,5 +166,5 @@ module.exports = {
/>
);
});
},
}
};

View file

@ -31,7 +31,8 @@ var dis = require("../../dispatcher");
var PAGINATE_SIZE = 20;
var INITIAL_SIZE = 100;
var ComponentBroker = require('../../ComponentBroker');
var ConferenceHandler = require("../../ConferenceHandler");
var CallHandler = require("../../CallHandler");
var Notifier = ComponentBroker.get('organisms/Notifier');
var tileTypes = {
@ -62,6 +63,7 @@ module.exports = {
MatrixClientPeg.get().on("Room.timeline", this.onRoomTimeline);
MatrixClientPeg.get().on("Room.name", this.onRoomName);
MatrixClientPeg.get().on("RoomMember.typing", this.onRoomMemberTyping);
MatrixClientPeg.get().on("RoomState.members", this.onRoomStateMember);
this.atBottom = true;
},
@ -78,6 +80,7 @@ module.exports = {
MatrixClientPeg.get().removeListener("Room.timeline", this.onRoomTimeline);
MatrixClientPeg.get().removeListener("Room.name", this.onRoomName);
MatrixClientPeg.get().removeListener("RoomMember.typing", this.onRoomMemberTyping);
MatrixClientPeg.get().removeListener("RoomState.members", this.onRoomStateMember);
}
},
@ -94,15 +97,20 @@ module.exports = {
this.forceUpdate();
break;
case 'call_state':
if (this.props.roomId !== payload.room_id) {
break;
}
// scroll to bottom
var messageWrapper = this.refs.messageWrapper;
if (messageWrapper) {
messageWrapper = messageWrapper.getDOMNode();
messageWrapper.scrollTop = messageWrapper.scrollHeight;
if (CallHandler.getCallForRoom(this.props.roomId)) {
// Call state has changed so we may be loading video elements
// which will obscure the message log.
// scroll to bottom
var messageWrapper = this.refs.messageWrapper;
if (messageWrapper) {
messageWrapper = messageWrapper.getDOMNode();
messageWrapper.scrollTop = messageWrapper.scrollHeight;
}
}
// possibly remove the conf call notification if we're now in
// the conf
this._updateConfCallNotification();
break;
}
},
@ -170,6 +178,42 @@ module.exports = {
this.forceUpdate();
},
onRoomStateMember: function(ev, state, member) {
if (member.roomId !== this.props.roomId ||
member.userId !== ConferenceHandler.getConferenceUserIdForRoom(member.roomId)) {
return;
}
this._updateConfCallNotification();
},
_updateConfCallNotification: function() {
var confMember = MatrixClientPeg.get().getRoom(this.props.roomId).getMember(
ConferenceHandler.getConferenceUserIdForRoom(this.props.roomId)
);
if (!confMember) {
return;
}
var confCall = CallHandler.getConferenceCall(confMember.roomId);
// A conf call notification should be displayed if there is an ongoing
// conf call but this cilent isn't a part of it.
this.setState({
displayConfCallNotification: (
(!confCall || confCall.call_state === "ended") &&
confMember.membership === "join"
)
});
},
onConferenceNotificationClick: function() {
dis.dispatch({
action: 'place_call',
type: "video",
room_id: this.props.roomId
});
},
componentDidMount: function() {
if (this.refs.messageWrapper) {
var messageWrapper = this.refs.messageWrapper.getDOMNode();
@ -183,6 +227,7 @@ module.exports = {
this.fillSpace();
}
this._updateConfCallNotification();
},
componentDidUpdate: function() {