Merge pull request #19 from matrix-org/bwindels/lltests2

Test all members are in memberlist with LL turned on

src/logbuffer.js

@@ -0,0 +1,30 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+module.exports = class LogBuffer {
+    constructor(page, eventName, eventMapper, reduceAsync=false, initialValue = "") {
+        this.buffer = initialValue;
+        page.on(eventName, (arg) => {
+            const result = eventMapper(arg);
+            if (reduceAsync) {
+                result.then((r) => this.buffer += r);
+            }
+            else {
+                this.buffer += result;
+            }
+        });
+    }
+}

src/logger.js

@@ -0,0 +1,62 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+module.exports = class Logger {
+    constructor(username) {
+        this.indent = 0;
+        this.username = username;
+        this.muted = false;
+    }
+
+    startGroup(description) {
+        if (!this.muted) {
+            const indent = " ".repeat(this.indent * 2);
+            console.log(`${indent} * ${this.username} ${description}:`);
+        }
+        this.indent += 1;
+        return this;
+    }
+
+    endGroup() {
+        this.indent -= 1;
+        return this;
+    }
+
+    step(description) {
+        if (!this.muted) {
+            const indent = " ".repeat(this.indent * 2);
+            process.stdout.write(`${indent} * ${this.username} ${description} ... `);
+        }
+        return this;
+    }
+
+    done(status = "done") {
+        if (!this.muted) {
+            process.stdout.write(status + "\n");
+        }
+        return this;
+    }
+
+    mute() {
+        this.muted = true;
+        return this;
+    }
+
+    unmute() {
+        this.muted = false;
+        return this;
+    }
+}

src/scenario.js

@@ -15,22 +15,12 @@ limitations under the License.
 */
 
 
-const {acceptDialogMaybe} = require('./tests/dialog');
-const signup = require('./tests/signup');
-const join = require('./tests/join');
-const sendMessage = require('./tests/send-message');
-const acceptInvite = require('./tests/accept-invite');
-const invite = require('./tests/invite');
-const {
-    receiveMessage,
-    checkTimelineContains,
-    scrollToTimelineTop
-} = require('./tests/timeline');
-const createRoom = require('./tests/create-room');
-const changeRoomSettings = require('./tests/room-settings');
-const acceptServerNoticesInviteAndConsent = require('./tests/server-notices-consent');
-const {enableLazyLoading, getE2EDeviceFromSettings} = require('./tests/settings');
-const verifyDeviceForUser = require("./tests/verify-device");
+const {range} = require('./util');
+const signup = require('./usecases/signup');
+const acceptServerNoticesInviteAndConsent = require('./usecases/server-notices-consent');
+const roomDirectoryScenarios = require('./scenarios/directory');
+const lazyLoadingScenarios = require('./scenarios/lazy-loading');
+const e2eEncryptionScenarios = require('./scenarios/e2e-encryption');
 
 module.exports = async function scenario(createSession, restCreator) {
     async function createUser(username) {
@@ -44,17 +34,9 @@ module.exports = async function scenario(createSession, restCreator) {
     const bob = await createUser("bob");
     const charlies = await createRestUsers(restCreator);
 
-    await createDirectoryRoomAndTalk(alice, bob);
-    await createE2ERoomAndTalk(alice, bob);
-    await aLazyLoadingTest(alice, bob, charlies);
-}
-
-function range(start, amount, step = 1) {
-    const r = [];
-    for (let i = 0; i < amount; ++i) {
-        r.push(start + (i * step));
-    }
-    return r;
+    await roomDirectoryScenarios(alice, bob);
+    await e2eEncryptionScenarios(alice, bob);
+    await lazyLoadingScenarios(alice, bob, charlies);
 }
 
 async function createRestUsers(restCreator) {
@@ -63,79 +45,3 @@ async function createRestUsers(restCreator) {
     await charlies.setDisplayName((s) => `Charly #${s.userName().split('-')[1]}`);
     return charlies;
 }
-
-async function createDirectoryRoomAndTalk(alice, bob) {
-    console.log(" creating a public room and join through directory:");
-    const room = 'test';
-    await createRoom(alice, room);
-    await changeRoomSettings(alice, {directory: true, visibility: "public_no_guests"});
-    await join(bob, room);
-    const bobMessage = "hi Alice!";
-    await sendMessage(bob, bobMessage);
-    await receiveMessage(alice, {sender: "bob", body: bobMessage});
-    const aliceMessage = "hi Bob, welcome!"
-    await sendMessage(alice, aliceMessage);
-    await receiveMessage(bob, {sender: "alice", body: aliceMessage});
-}
-
-async function createE2ERoomAndTalk(alice, bob) {
-    console.log(" creating an e2e encrypted room and join through invite:");
-    const room = "secrets";
-    await createRoom(bob, room);
-    await changeRoomSettings(bob, {encryption: true});
-    await invite(bob, "@alice:localhost");
-    await acceptInvite(alice, room);
-    const bobDevice = await getE2EDeviceFromSettings(bob);
-    // wait some time for the encryption warning dialog
-    // to appear after closing the settings
-    await bob.delay(1000);
-    await acceptDialogMaybe(bob, "encryption");
-    const aliceDevice = await getE2EDeviceFromSettings(alice);
-    // wait some time for the encryption warning dialog
-    // to appear after closing the settings
-    await alice.delay(1000);
-    await acceptDialogMaybe(alice, "encryption");
-    await verifyDeviceForUser(bob, "alice", aliceDevice);
-    await verifyDeviceForUser(alice, "bob", bobDevice);
-    const aliceMessage = "Guess what I just heard?!"
-    await sendMessage(alice, aliceMessage);
-    await receiveMessage(bob, {sender: "alice", body: aliceMessage, encrypted: true});
-    const bobMessage = "You've got to tell me!";
-    await sendMessage(bob, bobMessage);
-    await receiveMessage(alice, {sender: "bob", body: bobMessage, encrypted: true});
-}
-
-async function aLazyLoadingTest(alice, bob, charlies) {
-    console.log(" creating a room for lazy loading member scenarios:");
-    await enableLazyLoading(alice);
-    const room = "Lazy Loading Test";
-    const alias = "#lltest:localhost";
-    const charlyMsg1 = "hi bob!";
-    const charlyMsg2 = "how's it going??";
-    await createRoom(bob, room);
-    await changeRoomSettings(bob, {directory: true, visibility: "public_no_guests", alias});
-    // wait for alias to be set by server after clicking "save"
-    // so the charlies can join it.
-    await bob.delay(500);
-    const charlyMembers = await charlies.join(alias);
-    await charlyMembers.talk(charlyMsg1);
-    await charlyMembers.talk(charlyMsg2);
-    bob.log.step("sends 20 messages").mute();
-    for(let i = 20; i >= 1; --i) {
-        await sendMessage(bob, `I will only say this ${i} time(s)!`);
-    }
-    bob.log.unmute().done();
-    await join(alice, alias);
-    await scrollToTimelineTop(alice);
-    //alice should see 2 messages from every charly with
-    //the correct display name
-    const expectedMessages = [charlyMsg1, charlyMsg2].reduce((messages, msgText) => {
-        return charlies.sessions.reduce((messages, charly) => {
-            return messages.concat({
-                sender: charly.displayName(),
-                body: msgText,
-            });
-        }, messages);
-    }, []);
-    await checkTimelineContains(alice, expectedMessages, "Charly #1-10");
-}

src/scenarios/README.md

@@ -0,0 +1 @@
+scenarios contains the high-level playbook for the test suite

src/scenarios/directory.js

@@ -0,0 +1,36 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+
+const join = require('../usecases/join');
+const sendMessage = require('../usecases/send-message');
+const {receiveMessage} = require('../usecases/timeline');
+const createRoom = require('../usecases/create-room');
+const changeRoomSettings = require('../usecases/room-settings');
+
+module.exports = async function roomDirectoryScenarios(alice, bob) {
+    console.log(" creating a public room and join through directory:");
+    const room = 'test';
+    await createRoom(alice, room);
+    await changeRoomSettings(alice, {directory: true, visibility: "public_no_guests"});
+    await join(bob, room); //looks up room in directory
+    const bobMessage = "hi Alice!";
+    await sendMessage(bob, bobMessage);
+    await receiveMessage(alice, {sender: "bob", body: bobMessage});
+    const aliceMessage = "hi Bob, welcome!"
+    await sendMessage(alice, aliceMessage);
+    await receiveMessage(bob, {sender: "alice", body: aliceMessage});
+}

src/scenarios/e2e-encryption.js

@@ -0,0 +1,55 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+
+const {delay} = require('../util');
+const {acceptDialogMaybe} = require('../usecases/dialog');
+const join = require('../usecases/join');
+const sendMessage = require('../usecases/send-message');
+const acceptInvite = require('../usecases/accept-invite');
+const invite = require('../usecases/invite');
+const {receiveMessage} = require('../usecases/timeline');
+const createRoom = require('../usecases/create-room');
+const changeRoomSettings = require('../usecases/room-settings');
+const {getE2EDeviceFromSettings} = require('../usecases/settings');
+const {verifyDeviceForUser} = require('../usecases/memberlist');
+
+module.exports = async function e2eEncryptionScenarios(alice, bob) {
+    console.log(" creating an e2e encrypted room and join through invite:");
+    const room = "secrets";
+    await createRoom(bob, room);
+    await changeRoomSettings(bob, {encryption: true});
+    await invite(bob, "@alice:localhost");
+    await acceptInvite(alice, room);
+    const bobDevice = await getE2EDeviceFromSettings(bob);
+    // wait some time for the encryption warning dialog
+    // to appear after closing the settings
+    await delay(1000);
+    await acceptDialogMaybe(bob, "encryption");
+    const aliceDevice = await getE2EDeviceFromSettings(alice);
+    // wait some time for the encryption warning dialog
+    // to appear after closing the settings
+    await delay(1000);
+    await acceptDialogMaybe(alice, "encryption");
+    await verifyDeviceForUser(bob, "alice", aliceDevice);
+    await verifyDeviceForUser(alice, "bob", bobDevice);
+    const aliceMessage = "Guess what I just heard?!"
+    await sendMessage(alice, aliceMessage);
+    await receiveMessage(bob, {sender: "alice", body: aliceMessage, encrypted: true});
+    const bobMessage = "You've got to tell me!";
+    await sendMessage(bob, bobMessage);
+    await receiveMessage(alice, {sender: "bob", body: bobMessage, encrypted: true});
+}

src/scenarios/lazy-loading.js

@@ -0,0 +1,87 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+
+const {delay} = require('../util');
+const join = require('../usecases/join');
+const sendMessage = require('../usecases/send-message');
+const {
+    checkTimelineContains,
+    scrollToTimelineTop
+} = require('../usecases/timeline');
+const createRoom = require('../usecases/create-room');
+const {getMembersInMemberlist} = require('../usecases/memberlist');
+const changeRoomSettings = require('../usecases/room-settings');
+const {enableLazyLoading} = require('../usecases/settings');
+const assert = require('assert');
+
+module.exports = async function lazyLoadingScenarios(alice, bob, charlies) {
+    console.log(" creating a room for lazy loading member scenarios:");
+    await enableLazyLoading(alice);
+    await setupRoomWithBobAliceAndCharlies(alice, bob, charlies);
+    await checkPaginatedDisplayNames(alice, charlies);
+    await checkMemberList(alice, charlies);
+}
+
+const room = "Lazy Loading Test";
+const alias = "#lltest:localhost";
+const charlyMsg1 = "hi bob!";
+const charlyMsg2 = "how's it going??";
+
+async function setupRoomWithBobAliceAndCharlies(alice, bob, charlies) {
+    await createRoom(bob, room);
+    await changeRoomSettings(bob, {directory: true, visibility: "public_no_guests", alias});
+    // wait for alias to be set by server after clicking "save"
+    // so the charlies can join it.
+    await bob.delay(500);
+    const charlyMembers = await charlies.join(alias);
+    await charlyMembers.talk(charlyMsg1);
+    await charlyMembers.talk(charlyMsg2);
+    bob.log.step("sends 20 messages").mute();
+    for(let i = 20; i >= 1; --i) {
+        await sendMessage(bob, `I will only say this ${i} time(s)!`);
+    }
+    bob.log.unmute().done();
+    await join(alice, alias);
+}
+
+async function checkPaginatedDisplayNames(alice, charlies) {
+    await scrollToTimelineTop(alice);
+    //alice should see 2 messages from every charly with
+    //the correct display name
+    const expectedMessages = [charlyMsg1, charlyMsg2].reduce((messages, msgText) => {
+        return charlies.sessions.reduce((messages, charly) => {
+            return messages.concat({
+                sender: charly.displayName(),
+                body: msgText,
+            });
+        }, messages);
+    }, []);
+    await checkTimelineContains(alice, expectedMessages, "Charly #1-10");
+}
+
+async function checkMemberList(alice, charlies) {
+    alice.log.step("checks the memberlist contains herself, bob and all charlies");
+    const displayNames = (await getMembersInMemberlist(alice)).map((m) => m.displayName);
+    assert(displayNames.includes("alice"));
+    assert(displayNames.includes("bob"));
+    charlies.sessions.forEach((charly) => {
+        assert(displayNames.includes(charly.displayName()),
+            `${charly.displayName()} should be in the member list, ` +
+            `only have ${displayNames}`);
+    });
+    alice.log.done();
+}

src/session.js

@@ -15,68 +15,9 @@ limitations under the License.
 */
 
 const puppeteer = require('puppeteer');
-
-class LogBuffer {
-    constructor(page, eventName, eventMapper, reduceAsync=false, initialValue = "") {
-        this.buffer = initialValue;
-        page.on(eventName, (arg) => {
-            const result = eventMapper(arg);
-            if (reduceAsync) {
-                result.then((r) => this.buffer += r);
-            }
-            else {
-                this.buffer += result;
-            }
-        });
-    }
-}
-
-class Logger {
-    constructor(username) {
-        this.indent = 0;
-        this.username = username;
-        this.muted = false;
-    }
-
-    startGroup(description) {
-        if (!this.muted) {
-            const indent = " ".repeat(this.indent * 2);
-            console.log(`${indent} * ${this.username} ${description}:`);
-        }
-        this.indent += 1;
-        return this;
-    }
-
-    endGroup() {
-        this.indent -= 1;
-        return this;
-    }
-
-    step(description) {
-        if (!this.muted) {
-            const indent = " ".repeat(this.indent * 2);
-            process.stdout.write(`${indent} * ${this.username} ${description} ... `);
-        }
-        return this;
-    }
-
-    done(status = "done") {
-        if (!this.muted) {
-            process.stdout.write(status + "\n");
-        }
-        return this;
-    }
-
-    mute() {
-        this.muted = true;
-        return this;
-    }
-
-    unmute() {
-        this.muted = false;
-        return this;
-    }
-}
+const Logger = require('./logger');
+const LogBuffer = require('./logbuffer');
+const {delay} = require('./util');
 
 module.exports = class RiotSession {
     constructor(browser, page, username, riotserver, hsUrl) {
@@ -183,7 +124,7 @@ module.exports = class RiotSession {
         return await this.queryAll(selector);
     }
 
-    waitForReload(timeout = 5000) {
+    waitForReload(timeout = 10000) {
         return new Promise((resolve, reject) => {
             const timeoutHandle = setTimeout(() => {
                 this.browser.removeEventListener('domcontentloaded', callback);
@@ -229,7 +170,7 @@ module.exports = class RiotSession {
     }
 
     delay(ms) {
-        return new Promise(resolve => setTimeout(resolve, ms));
+        return delay(ms);
     }
 
     close() {

src/usecases/README.md

@@ -0,0 +1,2 @@
+use cases contains the detailed DOM interactions to perform a given use case, may also do some assertions.
+use cases are often used in multiple scenarios.

src/usecases/memberlist.js

@@ -16,16 +16,13 @@ limitations under the License.
 
 const assert = require('assert');
 
-module.exports = async function verifyDeviceForUser(session, name, expectedDevice) {
+module.exports.verifyDeviceForUser = async function(session, name, expectedDevice) {
     session.log.step(`verifies e2e device for ${name}`);
-    const memberNameElements = await session.queryAll(".mx_MemberList .mx_EntityTile_name");
-    const membersAndNames = await Promise.all(memberNameElements.map(async (el) => {
-        return [el, await session.innerText(el)];
-    }));
-    const matchingMember = membersAndNames.filter(([el, text]) => {
-        return text === name;
-    }).map(([el]) => el)[0];
-    await matchingMember.click();
+    const membersAndNames = await getMembersInMemberlist(session);
+    const matchingLabel = membersAndNames.filter((m) => {
+        return m.displayName === name;
+    }).map((m) => m.label)[0];
+    await matchingLabel.click();
     const firstVerifyButton = await session.waitAndQuery(".mx_MemberDeviceInfo_verify");
     await firstVerifyButton.click();
     const dialogCodeFields = await session.waitAndQueryAll(".mx_QuestionDialog code");
@@ -40,3 +37,12 @@ module.exports = async function verifyDeviceForUser(session, name, expectedDevic
     await closeMemberInfo.click();
     session.log.done();
 }
+
+async function getMembersInMemberlist(session) {
+    const memberNameElements = await session.waitAndQueryAll(".mx_MemberList .mx_EntityTile_name");
+    return Promise.all(memberNameElements.map(async (el) => {
+        return {label: el, displayName: await session.innerText(el)};
+    }));
+}
+
+module.exports.getMembersInMemberlist = getMembersInMemberlist;

src/usecases/timeline.js

@@ -46,6 +46,24 @@ module.exports.receiveMessage = async function(session, expectedMessage) {
     session.log.step(`receives message "${expectedMessage.body}" from ${expectedMessage.sender}`);
     // wait for a response to come in that contains the message
     // crude, but effective
+
+    async function getLastMessage() {
+        const lastTile = await getLastEventTile(session);
+        return getMessageFromEventTile(lastTile);
+    }
+
+    let lastMessage = null;
+    let isExpectedMessage = false;
+    try {
+        lastMessage = await getLastMessage();
+        isExpectedMessage = lastMessage &&
+            lastMessage.body === expectedMessage.body &&
+            lastMessage.sender === expectedMessage.sender;
+    } catch(ex) {}
+    // first try to see if the message is already the last message in the timeline
+    if (isExpectedMessage) {
+        assertMessage(lastMessage, expectedMessage);
+    } else {
         await session.page.waitForResponse(async (response) => {
             if (response.request().url().indexOf("/sync") === -1) {
                 return false;
@@ -60,9 +78,10 @@ module.exports.receiveMessage = async function(session, expectedMessage) {
         });
         // wait a bit for the incoming event to be rendered
         await session.delay(1000);
-    const lastTile = await getLastEventTile(session);
-    const foundMessage = await getMessageFromEventTile(lastTile);
-    assertMessage(foundMessage, expectedMessage);
+        lastMessage = await getLastMessage();
+        assertMessage(lastMessage, expectedMessage);
+    }
+
     session.log.done();
 }
 

src/util.js

@@ -0,0 +1,27 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+module.exports.range = function(start, amount, step = 1) {
+    const r = [];
+    for (let i = 0; i < amount; ++i) {
+        r.push(start + (i * step));
+    }
+    return r;
+}
+
+module.exports.delay = function(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}