Automated browser testing (#1415)

* Move automated api tests to api directory

* First pass at automated browser testing

.github/workflows/automated-browser.yml

@@ -0,0 +1,14 @@
+name: Automated browser tests
+on: [push, pull_request]
+jobs:
+  browser:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run browser tests
+        run: cd test/automated/browser && ./run.sh
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: screenshots-${{ github.run_id }}
+          path: test/automated/browser/screenshots/*.png

.github/workflows/automated-end-to-end-api.yaml

@@ -0,0 +1,21 @@
+name: Automated API tests
+
+on:
+  push:
+    paths-ignore:
+      - 'webroot/**'
+      - pkged.go
+  pull_request:
+    paths-ignore:
+      - 'webroot/**'
+      - pkged.go
+      
+jobs:
+  api:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run API tests
+        run: cd test/automated/api && ./run.sh
+
+  

.github/workflows/automated-end-to-end.yaml

@@ -1,16 +0,0 @@
-name: Automated end to end tests
-
-on:
-  push:
-    # branches:
-    #   - develop
-  pull_request:
-    branches: develop
-
-jobs:
-  Jest:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup and run
-        run: cd test/automated && ./run.sh

.gitignore

@@ -37,3 +37,4 @@ backup/
 !webroot/js/web_modules/**/dist
 !core/data
 test/test.db
+test/automated/browser/screenshots

test/automated/api/run.sh

@@ -15,7 +15,7 @@ if [ ! -d "ffmpeg" ]; then
   popd > /dev/null
 fi
 
-pushd ../.. > /dev/null
+pushd ../../.. > /dev/null
 
 # Build and run owncast from source
 go build -o owncast main.go pkged.go
@@ -27,7 +27,7 @@ sleep 5
 
 # Start streaming the test file over RTMP to
 # the local owncast instance.
-ffmpeg -hide_banner -loglevel panic -stream_loop -1 -re -i test.mp4 -vcodec libx264 -profile:v main -sc_threshold 0 -b:v 1300k -acodec copy -f flv rtmp://127.0.0.1/live/abc123 &
+ffmpeg -hide_banner -loglevel panic -stream_loop -1 -re -i ../test.mp4 -vcodec libx264 -profile:v main -sc_threshold 0 -b:v 1300k -acodec copy -f flv rtmp://127.0.0.1/live/abc123 &
 FFMPEG_PID=$!
 
 function finish {

test/automated/browser/README.md

@@ -0,0 +1,27 @@
+# Automated browser tests
+
+The tests currently address the following interfaces:
+
+1. The main web frontend of Owncast
+1. The embeddable video player
+1. The embeddable read-only chat
+1. the embeddable read-write chat
+
+Each have a set of test to make sure they load, have the expected elements on the screen, that API requests are successful, and that there are no errors being thrown in the console.
+
+The main web frontend additionally iterates its tests over a set of different device characteristics to verify mobile and tablet usage and goes through some interactive usage of the page such as changing their name and sending a chat message by clicking and typing.
+
+While it emulates the user agent, screen size, and touch features of different devices, they're still just a copy of Chromium running and not a true emulation of these other devices. So any "it breaks only on Safari" type bugs will not get caught.
+
+It can't actually play video, so anything specific about video playback cannot be verified with these tests.
+
+## Setup
+
+`npm install`
+
+## Run
+
+`./run.sh`
+## Screenshots
+
+After the tests finish a set of screenshots will be saved into the `screenshots` directory to aid in troubleshooting or sanity checking different viewport sizes. three

test/automated/browser/chat-embed.test.js

@@ -0,0 +1,33 @@
+const listenForErrors = require('./lib/errors.js').listenForErrors;
+const interactiveChatTest = require('./tests/chat.js').interactiveChatTest;
+
+describe('Chat read-write embed page', () => {
+  beforeAll(async () => {
+    await page.setViewport({ width: 600, height: 700 });
+    listenForErrors(browser, page);
+    await page.goto('http://localhost:5309/embed/chat/readwrite');
+  });
+
+  afterAll(async () => {
+    await page.waitForTimeout(3000);
+    await page.screenshot({ path: 'screenshots/screenshot_chat_embed.png', fullPage: true });
+  });
+
+  const newName = 'frontend-browser-embed-test-name-change';
+  const fakeMessage = 'this is a test chat message sent via the automated browser tests on the read/write chat embed page.'
+
+  interactiveChatTest(browser, page, newName, fakeMessage, 'desktop');
+});
+
+describe('Chat read-only embed page', () => {
+  beforeAll(async () => {
+    await page.setViewport({ width: 500, height: 700 });
+    listenForErrors(browser, page);
+    await page.goto('http://localhost:5309/embed/chat/readonly');
+  });
+
+  it('should have the messages container', async () => {
+    await page.waitForSelector('#messages-container');
+  });
+
+});

test/automated/browser/jest.config.json

@@ -0,0 +1,4 @@
+{
+  "name": "owncast browser tests",
+  "preset": "jest-puppeteer"
+}

test/automated/browser/lib/errors.js

@@ -0,0 +1,48 @@
+async function listenForErrors(browser, page) {
+  const ignoredErrors = [
+    'ERR_ABORTED',
+    'MEDIA_ERR_SRC_NOT_SUPPORTED',
+  ];
+
+  // Emitted when the page emits an error event (for example, the page crashes)
+  page.on('error', (error) => {
+    throw new Error(`❌ ${error}`);
+  });
+
+  browser.on('error', (error) => {
+    throw new Error(`❌ ${error}`);
+  });
+
+  // Emitted when a script within the page has uncaught exception
+  page.on('pageerror', (error) => {
+    throw new Error(`❌ ${error}`);
+  });
+
+  // Catch all failed requests like 4xx..5xx status codes
+  page.on('requestfailed', (request) => {
+    const ignoreError = ignoredErrors.some(e => request.failure().errorText.includes(e));
+    if (!ignoreError) {
+      throw new Error(
+        `❌ url: ${request.url()}, errText: ${
+          request.failure().errorText
+        }, method: ${request.method()}`
+      );
+    }
+  });
+
+  // Listen for console errors in the browser.
+  page.on('console', msg => {
+    const type = msg._type;
+    if (type !== 'error') {
+        return;
+    }
+
+    const ignoreError = ignoredErrors.some(e => msg._text.includes(e));
+    if (!ignoreError) {
+      throw new Error(`❌ ${msg._text}`);
+    }
+});
+
+}
+
+module.exports.listenForErrors = listenForErrors;

test/automated/browser/main.test.js

@@ -0,0 +1,46 @@
+const listenForErrors = require('./lib/errors.js').listenForErrors;
+const interactiveChatTest = require('./tests/chat.js').interactiveChatTest;
+const videoTest = require('./tests/video.js').videoTest;
+const puppeteer = require('puppeteer');
+
+const phone = puppeteer.devices['iPhone 11'];
+const tabletLandscape = puppeteer.devices['iPad landscape'];
+const tablet = puppeteer.devices['iPad Pro'];
+const desktop = {
+  name: 'desktop',
+  userAgent:
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36',
+  viewport: {
+    width: 1920,
+    height: 1080,
+    deviceScaleFactor: 1,
+    isMobile: false,
+    hasTouch: false,
+    isLandscape: true,
+  },
+};
+
+const devices = [desktop, phone, tablet, tabletLandscape];
+
+describe('Frontend web page', () => {
+    beforeAll(async () => {
+        listenForErrors(browser, page);
+        await page.goto('http://localhost:5309');
+        await page.waitForTimeout(3000);
+    });
+
+    devices.forEach(async function (device) {
+        const newName = 'frontend-browser-test-name-change-'+device.name;
+        const fakeMessage =
+          'this is a test chat message sent via the automated browser tests on the main web frontend from ' + device.name;
+    
+        interactiveChatTest(browser, page, newName, fakeMessage, device.name);
+        videoTest(browser, page);    
+
+        await page.waitForTimeout(2000);
+        await page.screenshot({
+            path: 'screenshots/screenshot_main-' + device.name + '.png',
+            fullPage: true,
+        });
+    });
+});

test/automated/browser/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "owncast-browser-tests",
+  "version": "1.0.0",
+  "description": "Automated browser testing for Owncast",
+  "main": "index.js",
+  "scripts": {
+    "test": "jest"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/owncast/owncast.git"
+  },
+  "author": "",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/owncast/owncast/issues"
+  },
+  "homepage": "https://github.com/owncast/owncast#readme",
+  "devDependencies": {
+    "jest": "^27.2.0",
+    "jest-puppeteer": "^5.0.4",
+    "puppeteer": "^9.1.1"
+  }
+}

test/automated/browser/run.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+TEMP_DB=$(mktemp)
+
+# Install the node test framework
+npm install --silent > /dev/null
+
+# Download a specific version of ffmpeg
+if [ ! -d "ffmpeg" ]; then
+  mkdir ffmpeg
+  pushd ffmpeg > /dev/null
+  curl -sL https://github.com/vot/ffbinaries-prebuilt/releases/download/v4.2.1/ffmpeg-4.2.1-linux-64.zip --output ffmpeg.zip > /dev/null
+  unzip -o ffmpeg.zip > /dev/null
+  PATH=$PATH:$(pwd)
+  popd > /dev/null
+fi
+
+pushd ../../.. > /dev/null
+
+# Build and run owncast from source
+go build -o owncast main.go pkged.go
+./owncast -rtmpport 9021 -webserverport 5309 -database $TEMP_DB &
+SERVER_PID=$!
+
+popd > /dev/null
+sleep 5
+
+# Start streaming the test file over RTMP to
+# the local owncast instance.
+ffmpeg -hide_banner -loglevel panic -stream_loop -1 -re -i ../test.mp4 -vcodec libx264 -profile:v main -sc_threshold 0 -b:v 1300k -acodec copy -f flv rtmp://127.0.0.1:9021/live/abc123 &
+FFMPEG_PID=$!
+
+function finish {
+  rm $TEMP_DB
+  kill $SERVER_PID $FFMPEG_PID
+}
+trap finish EXIT
+
+echo "Waiting..."
+sleep 15
+
+# Run the tests against the instance.
+npm test

test/automated/browser/tests/chat.js

@@ -0,0 +1,42 @@
+async function interactiveChatTest(browser, page, newName, chatMessage, device) {
+  it('should have the chat input', async () => {
+    await page.waitForSelector('#message-input');
+  });
+
+  it('should have the chat input enabled', async () => {
+    const isDisabled = await page.evaluate(
+      'document.querySelector("#message-input").getAttribute("disabled")'
+    );
+    expect(isDisabled).not.toBe('true');
+  });
+
+  it('should have the username label', async () => {
+    await page.waitForSelector('#username-display');
+  });
+
+  it('should allow changing the username on ' + device, async () => {
+    await page.waitForSelector('#username-display');
+    await page.evaluate(()=>document.querySelector('#username-display').click())
+
+    await page.waitForSelector('#username-change-input');
+    await page.evaluate(()=>document.querySelector('#username-change-input').click())
+    await page.type('#username-change-input', 'a new name');
+
+    await page.evaluate(()=>document.querySelector('#username-change-input').click())
+    await page.type('#username-change-input', newName);
+
+    await page.waitForSelector('#button-update-username');
+    await page.evaluate(()=>document.querySelector('#button-update-username').click())
+  });
+
+  it('should allow typing a chat message', async () => {
+    await page.waitForSelector('#message-input');
+    await page.evaluate(()=>document.querySelector('#message-input').click())
+    await page.waitForTimeout(1000);
+    await page.focus('#message-input')
+    await page.keyboard.type(chatMessage)
+    page.keyboard.press('Enter');
+  });
+}
+
+module.exports.interactiveChatTest = interactiveChatTest;

test/automated/browser/tests/video.js

@@ -0,0 +1,11 @@
+async function videoTest(browser, page) {
+  it('should have the video container element', async () => {
+    await page.waitForSelector('#video-container');
+  });
+
+  it('should have the stream info status bar', async () => {
+    await page.waitForSelector('#stream-info');
+  });
+}
+
+module.exports.videoTest = videoTest;

test/automated/browser/video-embed.test.js

@@ -0,0 +1,17 @@
+const listenForErrors = require('./lib/errors.js').listenForErrors;
+const videoTest = require('./tests/video.js').videoTest;
+
+describe('Video embed page', () => {
+  beforeAll(async () => {
+    await page.setViewport({ width: 1080, height: 720 });
+    listenForErrors(browser, page);
+    await page.goto('http://localhost:5309/embed/video');
+  });
+
+  afterAll(async () => {
+    await page.waitForTimeout(3000);
+    await page.screenshot({ path: 'screenshots/screenshot_video_embed.png', fullPage: true });
+  });
+
+  videoTest(browser, page);
+});

webroot/js/app.js

@@ -434,7 +434,11 @@ export default class App extends Component {
       this.setState({
         isPlaying: false,
       });
-      this.player.vjsPlayer.pause();
+      try {
+        this.player.vjsPlayer.pause();
+      } catch (err) {
+        console.warn(err);
+      }
     } else {
       this.setState({
         isPlaying: true,
@@ -447,11 +451,15 @@ export default class App extends Component {
     const muted = this.player.vjsPlayer.muted();
     const volume = this.player.vjsPlayer.volume();
 
-    if (volume === 0) {
-      this.player.vjsPlayer.volume(0.5);
-      this.player.vjsPlayer.muted(false);
-    } else {
-      this.player.vjsPlayer.muted(!muted);
+    try {
+      if (volume === 0) {
+        this.player.vjsPlayer.volume(0.5);
+        this.player.vjsPlayer.muted(false);
+      } else {
+        this.player.vjsPlayer.muted(!muted);
+      }
+    } catch (err) {
+      console.warn(err);
     }
   }
 

webroot/js/components/player.js

@@ -91,7 +91,11 @@ class OwncastPlayer {
     this.log('Start playing');
     const source = { ...VIDEO_SRC };
 
-    this.vjsPlayer.volume(getLocalStorage(PLAYER_VOLUME) || 1);
+    try {
+      this.vjsPlayer.volume(getLocalStorage(PLAYER_VOLUME) || 1);
+    } catch (err) {
+      console.warn(err);
+    }
     this.vjsPlayer.src(source);
     // this.vjsPlayer.play();
   }