From 86ad6de41ee455285695270242a5d0983b014899 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Damir=20Jeli=C4=87?= <poljar@termina.org.uk>
Date: Wed, 20 May 2020 11:01:56 +0200
Subject: [PATCH 1/2] EventIndex: Handle null tokens in the crawler loop as
 well.

This is similar to 5eb510387c9aa08a061d5fd17bed3efc104b48eb. But now the
checkpoint arrived during a crawl.
---
 src/indexing/EventIndex.js | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/src/indexing/EventIndex.js b/src/indexing/EventIndex.js
index 02151f8474..16b0183da1 100644
--- a/src/indexing/EventIndex.js
+++ b/src/indexing/EventIndex.js
@@ -489,14 +489,20 @@ export default class EventIndex extends EventEmitter {
                 return object;
             });
 
-            // Create a new checkpoint so we can continue crawling the room for
-            // messages.
-            const newCheckpoint = {
-                roomId: checkpoint.roomId,
-                token: res.end,
-                fullCrawl: checkpoint.fullCrawl,
-                direction: checkpoint.direction,
-            };
+            let newCheckpoint;
+
+            // The token can be null for some reason. Don't create a checkpoint
+            // in that case since adding it to the db will fail.
+            if (res.end) {
+                // Create a new checkpoint so we can continue crawling the room
+                // for messages.
+                newCheckpoint = {
+                    roomId: checkpoint.roomId,
+                    token: res.end,
+                    fullCrawl: checkpoint.fullCrawl,
+                    direction: checkpoint.direction,
+                };
+            }
 
             try {
                 for (let i = 0; i < redactionEvents.length; i++) {
@@ -506,6 +512,15 @@ export default class EventIndex extends EventEmitter {
 
                 const eventsAlreadyAdded = await indexManager.addHistoricEvents(
                     events, newCheckpoint, checkpoint);
+
+                // We didn't get a valid new checkpoint from the server, nothing
+                // to do here anymore.
+                if (!newCheckpoint) {
+                    console.log("EventIndex: The server didn't return a valid ",
+                                "new checkpoint, not continuing the crawl.", checkpoint);
+                    continue
+                }
+
                 // If all events were already indexed we assume that we catched
                 // up with our index and don't need to crawl the room further.
                 // Let us delete the checkpoint in that case, otherwise push

From db72c645bae492a5a3daba70695ef137ef07d485 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Damir=20Jeli=C4=87?= <poljar@termina.org.uk>
Date: Wed, 20 May 2020 11:14:37 +0200
Subject: [PATCH 2/2] EventIndex: Add a missing semicolon.

---
 src/indexing/EventIndex.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/indexing/EventIndex.js b/src/indexing/EventIndex.js
index 16b0183da1..d372c38405 100644
--- a/src/indexing/EventIndex.js
+++ b/src/indexing/EventIndex.js
@@ -518,7 +518,7 @@ export default class EventIndex extends EventEmitter {
                 if (!newCheckpoint) {
                     console.log("EventIndex: The server didn't return a valid ",
                                 "new checkpoint, not continuing the crawl.", checkpoint);
-                    continue
+                    continue;
                 }
 
                 // If all events were already indexed we assume that we catched