2019-11-12 15:29:07 +03:00
|
|
|
/*
|
2019-11-18 12:16:29 +03:00
|
|
|
Copyright 2019 The Matrix.org Foundation C.I.C.
|
2019-11-12 15:29:07 +03:00
|
|
|
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
you may not use this file except in compliance with the License.
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
See the License for the specific language governing permissions and
|
|
|
|
limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2019-11-19 14:52:12 +03:00
|
|
|
import PlatformPeg from "../PlatformPeg";
|
2019-12-21 00:13:46 +03:00
|
|
|
import {MatrixClientPeg} from "../MatrixClientPeg";
|
2021-03-19 05:50:34 +03:00
|
|
|
import {RoomMember} from 'matrix-js-sdk/src/models/room-member';
|
|
|
|
import {EventTimeline} from 'matrix-js-sdk/src/models/event-timeline';
|
2019-11-26 15:15:55 +03:00
|
|
|
import {sleep} from "../utils/promise";
|
2020-07-28 20:53:43 +03:00
|
|
|
import SettingsStore from "../settings/SettingsStore";
|
2020-01-20 19:43:55 +03:00
|
|
|
import {EventEmitter} from "events";
|
2020-07-28 20:53:43 +03:00
|
|
|
import {SettingLevel} from "../settings/SettingLevel";
|
2020-01-15 14:05:02 +03:00
|
|
|
|
2019-11-20 14:47:20 +03:00
|
|
|
/*
|
2019-11-12 15:29:07 +03:00
|
|
|
* Event indexing class that wraps the platform specific event indexing.
|
|
|
|
*/
|
2020-01-24 13:28:33 +03:00
|
|
|
export default class EventIndex extends EventEmitter {
|
2019-11-12 15:29:07 +03:00
|
|
|
constructor() {
|
2020-01-24 13:28:33 +03:00
|
|
|
super();
|
2019-11-13 17:57:12 +03:00
|
|
|
this.crawlerCheckpoints = [];
|
2019-11-26 15:25:34 +03:00
|
|
|
// The time in ms that the crawler will wait loop iterations if there
|
|
|
|
// have not been any checkpoints to consume in the last iteration.
|
|
|
|
this._crawlerIdleTime = 5000;
|
2019-11-13 18:35:26 +03:00
|
|
|
// The maximum number of events our crawler should fetch in a single
|
|
|
|
// crawl.
|
|
|
|
this._eventsPerCrawl = 100;
|
2019-11-18 17:04:22 +03:00
|
|
|
this._crawler = null;
|
2019-11-26 15:37:07 +03:00
|
|
|
this._currentCheckpoint = null;
|
2019-11-12 15:29:07 +03:00
|
|
|
}
|
|
|
|
|
2019-11-14 16:13:49 +03:00
|
|
|
async init() {
|
2019-11-13 14:25:16 +03:00
|
|
|
const indexManager = PlatformPeg.get().getEventIndexingManager();
|
2019-11-21 12:38:21 +03:00
|
|
|
|
|
|
|
this.crawlerCheckpoints = await indexManager.loadCheckpoints();
|
|
|
|
console.log("EventIndex: Loaded checkpoints", this.crawlerCheckpoints);
|
2019-11-19 12:46:18 +03:00
|
|
|
|
|
|
|
this.registerListeners();
|
|
|
|
}
|
|
|
|
|
2020-01-28 17:06:43 +03:00
|
|
|
/**
|
|
|
|
* Register event listeners that are necessary for the event index to work.
|
|
|
|
*/
|
2019-11-19 12:46:18 +03:00
|
|
|
registerListeners() {
|
|
|
|
const client = MatrixClientPeg.get();
|
|
|
|
|
2019-11-20 14:30:03 +03:00
|
|
|
client.on('sync', this.onSync);
|
|
|
|
client.on('Room.timeline', this.onRoomTimeline);
|
|
|
|
client.on('Event.decrypted', this.onEventDecrypted);
|
|
|
|
client.on('Room.timelineReset', this.onTimelineReset);
|
2020-03-12 13:42:02 +03:00
|
|
|
client.on('Room.redaction', this.onRedaction);
|
2020-06-08 17:43:20 +03:00
|
|
|
client.on('RoomState.events', this.onRoomStateEvent);
|
2019-11-19 12:46:18 +03:00
|
|
|
}
|
|
|
|
|
2020-01-28 17:06:43 +03:00
|
|
|
/**
|
|
|
|
* Remove the event index specific event listeners.
|
|
|
|
*/
|
2019-11-19 12:46:18 +03:00
|
|
|
removeListeners() {
|
|
|
|
const client = MatrixClientPeg.get();
|
|
|
|
if (client === null) return;
|
|
|
|
|
2019-11-20 14:30:03 +03:00
|
|
|
client.removeListener('sync', this.onSync);
|
|
|
|
client.removeListener('Room.timeline', this.onRoomTimeline);
|
|
|
|
client.removeListener('Event.decrypted', this.onEventDecrypted);
|
|
|
|
client.removeListener('Room.timelineReset', this.onTimelineReset);
|
2020-03-12 13:42:02 +03:00
|
|
|
client.removeListener('Room.redaction', this.onRedaction);
|
2020-06-08 17:43:20 +03:00
|
|
|
client.removeListener('RoomState.events', this.onRoomStateEvent);
|
2019-11-12 15:29:07 +03:00
|
|
|
}
|
|
|
|
|
2020-01-24 13:15:57 +03:00
|
|
|
/**
|
|
|
|
* Get crawler checkpoints for the encrypted rooms and store them in the index.
|
2020-01-23 13:02:44 +03:00
|
|
|
*/
|
|
|
|
async addInitialCheckpoints() {
|
2019-11-13 14:25:16 +03:00
|
|
|
const indexManager = PlatformPeg.get().getEventIndexingManager();
|
2020-01-23 13:02:44 +03:00
|
|
|
const client = MatrixClientPeg.get();
|
|
|
|
const rooms = client.getRooms();
|
2019-11-12 15:29:07 +03:00
|
|
|
|
2020-01-23 13:02:44 +03:00
|
|
|
const isRoomEncrypted = (room) => {
|
|
|
|
return client.isRoomEncrypted(room.roomId);
|
|
|
|
};
|
2019-11-12 15:29:07 +03:00
|
|
|
|
2020-01-24 13:15:57 +03:00
|
|
|
// We only care to crawl the encrypted rooms, non-encrypted
|
2020-01-23 13:02:44 +03:00
|
|
|
// rooms can use the search provided by the homeserver.
|
|
|
|
const encryptedRooms = rooms.filter(isRoomEncrypted);
|
2019-11-12 15:29:07 +03:00
|
|
|
|
2020-01-23 13:02:44 +03:00
|
|
|
console.log("EventIndex: Adding initial crawler checkpoints");
|
2019-11-12 15:29:07 +03:00
|
|
|
|
2020-01-23 13:02:44 +03:00
|
|
|
// Gather the prev_batch tokens and create checkpoints for
|
|
|
|
// our message crawler.
|
|
|
|
await Promise.all(encryptedRooms.map(async (room) => {
|
|
|
|
const timeline = room.getLiveTimeline();
|
|
|
|
const token = timeline.getPaginationToken("b");
|
2019-11-12 15:29:07 +03:00
|
|
|
|
2020-01-23 13:02:44 +03:00
|
|
|
const backCheckpoint = {
|
|
|
|
roomId: room.roomId,
|
|
|
|
token: token,
|
|
|
|
direction: "b",
|
2020-04-01 16:17:59 +03:00
|
|
|
fullCrawl: true,
|
2019-11-12 15:29:07 +03:00
|
|
|
};
|
|
|
|
|
2020-01-23 13:02:44 +03:00
|
|
|
const forwardCheckpoint = {
|
|
|
|
roomId: room.roomId,
|
|
|
|
token: token,
|
|
|
|
direction: "f",
|
|
|
|
};
|
2019-11-12 15:29:07 +03:00
|
|
|
|
2020-01-31 18:37:30 +03:00
|
|
|
try {
|
|
|
|
if (backCheckpoint.token) {
|
|
|
|
await indexManager.addCrawlerCheckpoint(backCheckpoint);
|
|
|
|
this.crawlerCheckpoints.push(backCheckpoint);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (forwardCheckpoint.token) {
|
|
|
|
await indexManager.addCrawlerCheckpoint(forwardCheckpoint);
|
|
|
|
this.crawlerCheckpoints.push(forwardCheckpoint);
|
|
|
|
}
|
|
|
|
} catch (e) {
|
2021-04-27 18:23:27 +03:00
|
|
|
console.log(
|
|
|
|
"EventIndex: Error adding initial checkpoints for room",
|
|
|
|
room.roomId,
|
|
|
|
backCheckpoint,
|
|
|
|
forwardCheckpoint,
|
|
|
|
e,
|
|
|
|
);
|
2020-01-31 18:37:30 +03:00
|
|
|
}
|
2020-01-23 13:02:44 +03:00
|
|
|
}));
|
|
|
|
}
|
2019-11-12 15:29:07 +03:00
|
|
|
|
2020-01-28 18:13:56 +03:00
|
|
|
/*
|
2020-01-28 17:06:43 +03:00
|
|
|
* The sync event listener.
|
|
|
|
*
|
|
|
|
* The listener has two cases:
|
|
|
|
* - First sync after start up, check if the index is empty, add
|
|
|
|
* initial checkpoints, if so. Start the crawler background task.
|
|
|
|
* - Every other sync, tell the event index to commit all the queued up
|
|
|
|
* live events
|
|
|
|
*/
|
2020-01-23 13:02:44 +03:00
|
|
|
onSync = async (state, prevState, data) => {
|
|
|
|
const indexManager = PlatformPeg.get().getEventIndexingManager();
|
2019-11-12 15:29:07 +03:00
|
|
|
|
2020-01-23 13:02:44 +03:00
|
|
|
if (prevState === "PREPARED" && state === "SYNCING") {
|
2020-08-03 18:02:26 +03:00
|
|
|
// If our indexer is empty we're most likely running Element the
|
2019-11-12 15:29:07 +03:00
|
|
|
// first time with indexing support or running it with an
|
|
|
|
// initial sync. Add checkpoints to crawl our encrypted rooms.
|
2019-11-13 14:25:16 +03:00
|
|
|
const eventIndexWasEmpty = await indexManager.isEventIndexEmpty();
|
2020-01-23 13:02:44 +03:00
|
|
|
if (eventIndexWasEmpty) await this.addInitialCheckpoints();
|
2019-11-12 15:29:07 +03:00
|
|
|
|
|
|
|
this.startCrawler();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (prevState === "SYNCING" && state === "SYNCING") {
|
|
|
|
// A sync was done, presumably we queued up some live events,
|
|
|
|
// commit them now.
|
2019-11-13 14:25:16 +03:00
|
|
|
await indexManager.commitLiveEvents();
|
2019-11-12 15:29:07 +03:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-28 18:13:56 +03:00
|
|
|
/*
|
2020-01-28 17:06:43 +03:00
|
|
|
* The Room.timeline listener.
|
|
|
|
*
|
|
|
|
* This listener waits for live events in encrypted rooms, if they are
|
|
|
|
* decrypted or unencrypted we queue them to be added to the index,
|
|
|
|
* otherwise we save their event id and wait for them in the Event.decrypted
|
|
|
|
* listener.
|
|
|
|
*/
|
2019-11-20 14:30:03 +03:00
|
|
|
onRoomTimeline = async (ev, room, toStartOfTimeline, removed, data) => {
|
2021-05-18 13:41:20 +03:00
|
|
|
const client = MatrixClientPeg.get();
|
|
|
|
|
2019-11-12 15:29:07 +03:00
|
|
|
// We only index encrypted rooms locally.
|
2021-05-18 13:41:20 +03:00
|
|
|
if (!client.isRoomEncrypted(room.roomId)) return;
|
2019-11-12 15:29:07 +03:00
|
|
|
|
|
|
|
// If it isn't a live event or if it's redacted there's nothing to
|
|
|
|
// do.
|
|
|
|
if (toStartOfTimeline || !data || !data.liveEvent
|
|
|
|
|| ev.isRedacted()) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2021-05-18 15:01:38 +03:00
|
|
|
await client.decryptEventIfNeeded(ev);
|
2021-05-10 17:19:46 +03:00
|
|
|
|
|
|
|
await this.addLiveEventToIndex(ev);
|
2019-11-12 15:29:07 +03:00
|
|
|
}
|
|
|
|
|
2020-06-08 17:43:20 +03:00
|
|
|
onRoomStateEvent = async (ev, state) => {
|
|
|
|
if (!MatrixClientPeg.get().isRoomEncrypted(state.roomId)) return;
|
|
|
|
|
|
|
|
if (ev.getType() === "m.room.encryption" && !await this.isRoomIndexed(state.roomId)) {
|
2020-06-08 18:30:26 +03:00
|
|
|
console.log("EventIndex: Adding a checkpoint for a newly encrypted room", state.roomId);
|
2020-06-08 17:43:20 +03:00
|
|
|
this.addRoomCheckpoint(state.roomId, true);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-28 18:13:56 +03:00
|
|
|
/*
|
2020-01-28 17:06:43 +03:00
|
|
|
* The Event.decrypted listener.
|
|
|
|
*
|
|
|
|
* Checks if the event was marked for addition in the Room.timeline
|
|
|
|
* listener, if so queues it up to be added to the index.
|
|
|
|
*/
|
2019-11-20 14:30:03 +03:00
|
|
|
onEventDecrypted = async (ev, err) => {
|
2019-11-12 15:29:07 +03:00
|
|
|
// If the event isn't in our live event set, ignore it.
|
|
|
|
if (err) return;
|
|
|
|
await this.addLiveEventToIndex(ev);
|
|
|
|
}
|
|
|
|
|
2020-03-12 13:42:02 +03:00
|
|
|
/*
|
|
|
|
* The Room.redaction listener.
|
|
|
|
*
|
|
|
|
* Removes a redacted event from our event index.
|
|
|
|
*/
|
|
|
|
onRedaction = async (ev, room) => {
|
|
|
|
// We only index encrypted rooms locally.
|
|
|
|
if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return;
|
|
|
|
const indexManager = PlatformPeg.get().getEventIndexingManager();
|
|
|
|
|
|
|
|
try {
|
|
|
|
await indexManager.deleteEvent(ev.getAssociatedId());
|
|
|
|
} catch (e) {
|
|
|
|
console.log("EventIndex: Error deleting event from index", e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-28 18:13:56 +03:00
|
|
|
/*
|
2020-01-28 17:06:43 +03:00
|
|
|
* The Room.timelineReset listener.
|
|
|
|
*
|
|
|
|
* Listens for timeline resets that are caused by a limited timeline to
|
|
|
|
* re-add checkpoints for rooms that need to be crawled again.
|
|
|
|
*/
|
|
|
|
onTimelineReset = async (room, timelineSet, resetAllTimelines) => {
|
|
|
|
if (room === null) return;
|
|
|
|
if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return;
|
|
|
|
|
2020-05-21 13:35:47 +03:00
|
|
|
console.log("EventIndex: Adding a checkpoint because of a limited timeline",
|
|
|
|
room.roomId);
|
2020-01-28 17:06:43 +03:00
|
|
|
|
2020-05-21 11:10:15 +03:00
|
|
|
this.addRoomCheckpoint(room.roomId, false);
|
2020-01-28 17:06:43 +03:00
|
|
|
}
|
|
|
|
|
2020-02-18 16:13:08 +03:00
|
|
|
/**
|
|
|
|
* Check if an event should be added to the event index.
|
|
|
|
*
|
|
|
|
* Most notably we filter events for which decryption failed, are redacted
|
|
|
|
* or aren't of a type that we know how to index.
|
|
|
|
*
|
|
|
|
* @param {MatrixEvent} ev The event that should checked.
|
2020-02-18 16:45:03 +03:00
|
|
|
* @returns {bool} Returns true if the event can be indexed, false
|
|
|
|
* otherwise.
|
2020-02-18 16:13:08 +03:00
|
|
|
*/
|
|
|
|
isValidEvent(ev) {
|
2020-02-19 16:36:21 +03:00
|
|
|
const isUsefulType = ["m.room.message", "m.room.name", "m.room.topic"].includes(ev.getType());
|
|
|
|
const validEventType = isUsefulType && !ev.isRedacted() && !ev.isDecryptionFailure();
|
2020-02-18 16:13:08 +03:00
|
|
|
|
|
|
|
let validMsgType = true;
|
2020-04-20 11:02:40 +03:00
|
|
|
let hasContentValue = true;
|
2020-02-18 16:13:08 +03:00
|
|
|
|
|
|
|
if (ev.getType() === "m.room.message" && !ev.isRedacted()) {
|
|
|
|
// Expand this if there are more invalid msgtypes.
|
|
|
|
const msgtype = ev.getContent().msgtype;
|
|
|
|
|
|
|
|
if (!msgtype) validMsgType = false;
|
|
|
|
else validMsgType = !msgtype.startsWith("m.key.verification");
|
2020-04-20 11:02:40 +03:00
|
|
|
|
2020-04-20 11:10:16 +03:00
|
|
|
if (!ev.getContent().body) hasContentValue = false;
|
2020-04-20 11:02:40 +03:00
|
|
|
} else if (ev.getType() === "m.room.topic" && !ev.isRedacted()) {
|
|
|
|
if (!ev.getContent().topic) hasContentValue = false;
|
|
|
|
} else if (ev.getType() === "m.room.name" && !ev.isRedacted()) {
|
|
|
|
if (!ev.getContent().name) hasContentValue = false;
|
2020-02-18 16:13:08 +03:00
|
|
|
}
|
|
|
|
|
2020-04-20 11:02:40 +03:00
|
|
|
return validEventType && validMsgType && hasContentValue;
|
2020-02-18 16:13:08 +03:00
|
|
|
}
|
|
|
|
|
2020-06-09 14:53:27 +03:00
|
|
|
eventToJson(ev) {
|
|
|
|
const jsonEvent = ev.toJSON();
|
|
|
|
const e = ev.isEncrypted() ? jsonEvent.decrypted : jsonEvent;
|
|
|
|
|
|
|
|
if (ev.isEncrypted()) {
|
|
|
|
// Let us store some additional data so we can re-verify the event.
|
|
|
|
// The js-sdk checks if an event is encrypted using the algorithm,
|
|
|
|
// the sender key and ed25519 signing key are used to find the
|
|
|
|
// correct device that sent the event which allows us to check the
|
|
|
|
// verification state of the event, either directly or using cross
|
|
|
|
// signing.
|
|
|
|
e.curve25519Key = ev.getSenderKey();
|
|
|
|
e.ed25519Key = ev.getClaimedEd25519Key();
|
|
|
|
e.algorithm = ev.getWireContent().algorithm;
|
|
|
|
e.forwardingCurve25519KeyChain = ev.getForwardingCurve25519KeyChain();
|
|
|
|
} else {
|
|
|
|
// Make sure that unencrypted events don't contain any of that data,
|
|
|
|
// despite what the server might give to us.
|
|
|
|
delete e.curve25519Key;
|
|
|
|
delete e.ed25519Key;
|
|
|
|
delete e.algorithm;
|
|
|
|
delete e.forwardingCurve25519KeyChain;
|
|
|
|
}
|
|
|
|
|
|
|
|
return e;
|
|
|
|
}
|
|
|
|
|
2020-01-28 17:06:43 +03:00
|
|
|
/**
|
|
|
|
* Queue up live events to be added to the event index.
|
|
|
|
*
|
|
|
|
* @param {MatrixEvent} ev The event that should be added to the index.
|
|
|
|
*/
|
2019-11-12 15:29:07 +03:00
|
|
|
async addLiveEventToIndex(ev) {
|
2019-11-13 14:25:16 +03:00
|
|
|
const indexManager = PlatformPeg.get().getEventIndexingManager();
|
2019-11-12 15:29:07 +03:00
|
|
|
|
2020-02-18 16:13:08 +03:00
|
|
|
if (!this.isValidEvent(ev)) return;
|
2019-11-12 15:29:07 +03:00
|
|
|
|
2020-06-09 14:53:27 +03:00
|
|
|
const e = this.eventToJson(ev);
|
2020-01-15 12:41:33 +03:00
|
|
|
|
2019-11-12 15:29:07 +03:00
|
|
|
const profile = {
|
|
|
|
displayname: ev.sender.rawDisplayName,
|
|
|
|
avatar_url: ev.sender.getMxcAvatarUrl(),
|
|
|
|
};
|
|
|
|
|
2020-05-21 11:10:46 +03:00
|
|
|
await indexManager.addEventToIndex(e, profile);
|
2019-11-12 15:29:07 +03:00
|
|
|
}
|
|
|
|
|
2020-01-28 17:06:43 +03:00
|
|
|
/**
|
|
|
|
* Emmit that the crawler has changed the checkpoint that it's currently
|
|
|
|
* handling.
|
|
|
|
*/
|
2020-01-20 19:43:55 +03:00
|
|
|
emitNewCheckpoint() {
|
2020-01-24 13:28:33 +03:00
|
|
|
this.emit("changedCheckpoint", this.currentRoom());
|
2020-01-20 19:43:55 +03:00
|
|
|
}
|
2019-11-12 15:29:07 +03:00
|
|
|
|
2020-05-21 11:10:15 +03:00
|
|
|
async addEventsFromLiveTimeline(timeline) {
|
2020-05-21 13:35:47 +03:00
|
|
|
const events = timeline.getEvents();
|
2020-05-21 11:10:15 +03:00
|
|
|
|
|
|
|
for (let i = 0; i < events.length; i++) {
|
|
|
|
const ev = events[i];
|
|
|
|
await this.addLiveEventToIndex(ev);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
async addRoomCheckpoint(roomId, fullCrawl = false) {
|
|
|
|
const indexManager = PlatformPeg.get().getEventIndexingManager();
|
|
|
|
const client = MatrixClientPeg.get();
|
|
|
|
const room = client.getRoom(roomId);
|
|
|
|
|
|
|
|
if (!room) return;
|
|
|
|
|
|
|
|
const timeline = room.getLiveTimeline();
|
2020-05-21 13:35:47 +03:00
|
|
|
const token = timeline.getPaginationToken("b");
|
2020-05-21 11:10:15 +03:00
|
|
|
|
2020-05-21 13:35:47 +03:00
|
|
|
if (!token) {
|
2020-05-21 11:10:15 +03:00
|
|
|
// The room doesn't contain any tokens, meaning the live timeline
|
|
|
|
// contains all the events, add those to the index.
|
|
|
|
await this.addEventsFromLiveTimeline(timeline);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
const checkpoint = {
|
|
|
|
roomId: room.roomId,
|
|
|
|
token: token,
|
|
|
|
fullCrawl: fullCrawl,
|
|
|
|
direction: "b",
|
|
|
|
};
|
|
|
|
|
|
|
|
console.log("EventIndex: Adding checkpoint", checkpoint);
|
|
|
|
|
2020-05-21 13:35:47 +03:00
|
|
|
try {
|
2020-05-21 11:10:15 +03:00
|
|
|
await indexManager.addCrawlerCheckpoint(checkpoint);
|
|
|
|
} catch (e) {
|
2021-04-27 18:23:27 +03:00
|
|
|
console.log(
|
|
|
|
"EventIndex: Error adding new checkpoint for room",
|
|
|
|
room.roomId,
|
|
|
|
checkpoint,
|
|
|
|
e,
|
|
|
|
);
|
2020-05-21 11:10:15 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
this.crawlerCheckpoints.push(checkpoint);
|
|
|
|
}
|
|
|
|
|
2020-01-28 17:06:43 +03:00
|
|
|
/**
|
|
|
|
* The main crawler loop.
|
|
|
|
*
|
|
|
|
* Goes through crawlerCheckpoints and fetches events from the server to be
|
|
|
|
* added to the EventIndex.
|
|
|
|
*
|
|
|
|
* If a /room/{roomId}/messages request doesn't contain any events, stop the
|
|
|
|
* crawl, otherwise create a new checkpoint and push it to the
|
|
|
|
* crawlerCheckpoints queue so we go through them in a round-robin way.
|
|
|
|
*/
|
2019-11-18 17:04:22 +03:00
|
|
|
async crawlerFunc() {
|
2019-11-12 15:29:07 +03:00
|
|
|
let cancelled = false;
|
|
|
|
|
|
|
|
const client = MatrixClientPeg.get();
|
2019-11-13 14:25:16 +03:00
|
|
|
const indexManager = PlatformPeg.get().getEventIndexingManager();
|
2019-11-12 15:29:07 +03:00
|
|
|
|
2019-11-18 17:04:22 +03:00
|
|
|
this._crawler = {};
|
|
|
|
|
|
|
|
this._crawler.cancel = () => {
|
2019-11-12 15:29:07 +03:00
|
|
|
cancelled = true;
|
|
|
|
};
|
|
|
|
|
2019-11-26 15:25:34 +03:00
|
|
|
let idle = false;
|
|
|
|
|
2019-11-12 15:29:07 +03:00
|
|
|
while (!cancelled) {
|
2020-01-31 20:44:52 +03:00
|
|
|
let sleepTime = SettingsStore.getValueAt(SettingLevel.DEVICE, 'crawlerSleepTime');
|
2019-11-26 15:25:34 +03:00
|
|
|
|
|
|
|
// Don't let the user configure a lower sleep time than 100 ms.
|
|
|
|
sleepTime = Math.max(sleepTime, 100);
|
|
|
|
|
|
|
|
if (idle) {
|
|
|
|
sleepTime = this._crawlerIdleTime;
|
|
|
|
}
|
|
|
|
|
2020-01-20 19:43:55 +03:00
|
|
|
if (this._currentCheckpoint !== null) {
|
|
|
|
this._currentCheckpoint = null;
|
|
|
|
this.emitNewCheckpoint();
|
|
|
|
}
|
2019-11-26 15:37:07 +03:00
|
|
|
|
2019-11-26 15:25:34 +03:00
|
|
|
await sleep(sleepTime);
|
2019-11-12 15:29:07 +03:00
|
|
|
|
|
|
|
if (cancelled) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2019-11-13 17:57:12 +03:00
|
|
|
const checkpoint = this.crawlerCheckpoints.shift();
|
2019-11-12 15:29:07 +03:00
|
|
|
|
|
|
|
/// There is no checkpoint available currently, one may appear if
|
|
|
|
// a sync with limited room timelines happens, so go back to sleep.
|
|
|
|
if (checkpoint === undefined) {
|
2019-11-26 15:25:34 +03:00
|
|
|
idle = true;
|
2019-11-12 15:29:07 +03:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2019-11-26 15:37:07 +03:00
|
|
|
this._currentCheckpoint = checkpoint;
|
2020-01-20 19:43:55 +03:00
|
|
|
this.emitNewCheckpoint();
|
2019-11-26 15:37:07 +03:00
|
|
|
|
2019-11-26 15:25:34 +03:00
|
|
|
idle = false;
|
|
|
|
|
2019-11-12 15:29:07 +03:00
|
|
|
// We have a checkpoint, let us fetch some messages, again, very
|
2019-11-18 12:35:57 +03:00
|
|
|
// conservatively to not bother our homeserver too much.
|
2020-04-15 18:37:32 +03:00
|
|
|
const eventMapper = client.getEventMapper({preventReEmit: true});
|
2019-11-12 15:29:07 +03:00
|
|
|
// TODO we need to ensure to use member lazy loading with this
|
|
|
|
// request so we get the correct profiles.
|
|
|
|
let res;
|
|
|
|
|
|
|
|
try {
|
2021-06-02 06:36:28 +03:00
|
|
|
res = await client.createMessagesRequest(
|
2019-11-13 18:35:26 +03:00
|
|
|
checkpoint.roomId, checkpoint.token, this._eventsPerCrawl,
|
2019-11-12 15:29:07 +03:00
|
|
|
checkpoint.direction);
|
|
|
|
} catch (e) {
|
2020-03-13 16:12:48 +03:00
|
|
|
if (e.httpStatus === 403) {
|
|
|
|
console.log("EventIndex: Removing checkpoint as we don't have ",
|
2021-04-27 18:23:27 +03:00
|
|
|
"permissions to fetch messages from this room.", checkpoint);
|
2020-03-13 16:12:48 +03:00
|
|
|
try {
|
|
|
|
await indexManager.removeCrawlerCheckpoint(checkpoint);
|
|
|
|
} catch (e) {
|
|
|
|
console.log("EventIndex: Error removing checkpoint", checkpoint, e);
|
|
|
|
// We don't push the checkpoint here back, it will
|
|
|
|
// hopefully be removed after a restart. But let us
|
|
|
|
// ignore it for now as we don't want to hammer the
|
|
|
|
// endpoint.
|
|
|
|
}
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2020-05-26 21:47:30 +03:00
|
|
|
console.log("EventIndex: Error crawling using checkpoint:", checkpoint, ",", e);
|
2019-11-13 17:57:12 +03:00
|
|
|
this.crawlerCheckpoints.push(checkpoint);
|
2019-11-13 14:25:16 +03:00
|
|
|
continue;
|
2019-11-12 15:29:07 +03:00
|
|
|
}
|
|
|
|
|
2020-01-24 13:46:46 +03:00
|
|
|
if (cancelled) {
|
|
|
|
this.crawlerCheckpoints.push(checkpoint);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2019-11-12 15:29:07 +03:00
|
|
|
if (res.chunk.length === 0) {
|
2019-11-13 17:26:27 +03:00
|
|
|
console.log("EventIndex: Done with the checkpoint", checkpoint);
|
2019-11-12 15:29:07 +03:00
|
|
|
// We got to the start/end of our timeline, lets just
|
|
|
|
// delete our checkpoint and go back to sleep.
|
2020-01-31 17:37:06 +03:00
|
|
|
try {
|
|
|
|
await indexManager.removeCrawlerCheckpoint(checkpoint);
|
|
|
|
} catch (e) {
|
|
|
|
console.log("EventIndex: Error removing checkpoint", checkpoint, e);
|
|
|
|
}
|
2019-11-12 15:29:07 +03:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Convert the plain JSON events into Matrix events so they get
|
|
|
|
// decrypted if necessary.
|
|
|
|
const matrixEvents = res.chunk.map(eventMapper);
|
|
|
|
let stateEvents = [];
|
|
|
|
if (res.state !== undefined) {
|
|
|
|
stateEvents = res.state.map(eventMapper);
|
|
|
|
}
|
|
|
|
|
|
|
|
const profiles = {};
|
|
|
|
|
|
|
|
stateEvents.forEach(ev => {
|
|
|
|
if (ev.event.content &&
|
|
|
|
ev.event.content.membership === "join") {
|
|
|
|
profiles[ev.event.sender] = {
|
|
|
|
displayname: ev.event.content.displayname,
|
|
|
|
avatar_url: ev.event.content.avatar_url,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
});
|
|
|
|
|
2021-05-10 17:19:46 +03:00
|
|
|
const decryptionPromises = matrixEvents
|
|
|
|
.filter(event => event.isEncrypted())
|
|
|
|
.map(event => {
|
2021-05-18 15:01:38 +03:00
|
|
|
return client.decryptEventIfNeeded(event, {
|
2021-05-18 13:41:20 +03:00
|
|
|
isRetry: true,
|
|
|
|
emit: false,
|
|
|
|
});
|
2021-05-10 17:19:46 +03:00
|
|
|
});
|
2019-11-12 15:29:07 +03:00
|
|
|
|
|
|
|
// Let us wait for all the events to get decrypted.
|
|
|
|
await Promise.all(decryptionPromises);
|
|
|
|
|
2019-11-18 17:04:44 +03:00
|
|
|
// TODO if there are no events at this point we're missing a lot
|
|
|
|
// decryption keys, do we want to retry this checkpoint at a later
|
2019-11-12 15:29:07 +03:00
|
|
|
// stage?
|
2020-02-18 16:13:08 +03:00
|
|
|
const filteredEvents = matrixEvents.filter(this.isValidEvent);
|
2019-11-12 15:29:07 +03:00
|
|
|
|
2020-03-12 13:42:02 +03:00
|
|
|
// Collect the redaction events so we can delete the redacted events
|
|
|
|
// from the index.
|
|
|
|
const redactionEvents = matrixEvents.filter((ev) => {
|
|
|
|
return ev.getType() === "m.room.redaction";
|
|
|
|
});
|
|
|
|
|
2019-11-13 17:26:27 +03:00
|
|
|
// Let us convert the events back into a format that EventIndex can
|
2019-11-12 15:29:07 +03:00
|
|
|
// consume.
|
|
|
|
const events = filteredEvents.map((ev) => {
|
2020-06-09 14:53:27 +03:00
|
|
|
const e = this.eventToJson(ev);
|
2019-11-12 15:29:07 +03:00
|
|
|
|
|
|
|
let profile = {};
|
|
|
|
if (e.sender in profiles) profile = profiles[e.sender];
|
|
|
|
const object = {
|
|
|
|
event: e,
|
|
|
|
profile: profile,
|
|
|
|
};
|
|
|
|
return object;
|
|
|
|
});
|
|
|
|
|
2020-05-20 12:01:56 +03:00
|
|
|
let newCheckpoint;
|
|
|
|
|
|
|
|
// The token can be null for some reason. Don't create a checkpoint
|
|
|
|
// in that case since adding it to the db will fail.
|
|
|
|
if (res.end) {
|
|
|
|
// Create a new checkpoint so we can continue crawling the room
|
|
|
|
// for messages.
|
|
|
|
newCheckpoint = {
|
|
|
|
roomId: checkpoint.roomId,
|
|
|
|
token: res.end,
|
|
|
|
fullCrawl: checkpoint.fullCrawl,
|
|
|
|
direction: checkpoint.direction,
|
|
|
|
};
|
|
|
|
}
|
2019-11-12 15:29:07 +03:00
|
|
|
|
|
|
|
try {
|
2020-03-12 14:20:46 +03:00
|
|
|
for (let i = 0; i < redactionEvents.length; i++) {
|
2020-03-12 14:15:46 +03:00
|
|
|
const ev = redactionEvents[i];
|
2020-05-26 22:27:45 +03:00
|
|
|
const eventId = ev.getAssociatedId();
|
|
|
|
|
|
|
|
if (eventId) {
|
|
|
|
await indexManager.deleteEvent(eventId);
|
|
|
|
} else {
|
2020-05-26 22:48:10 +03:00
|
|
|
console.warn("EventIndex: Redaction event doesn't contain a valid associated event id", ev);
|
2020-05-26 22:27:45 +03:00
|
|
|
}
|
2020-03-12 13:42:02 +03:00
|
|
|
}
|
|
|
|
|
2019-11-13 14:25:16 +03:00
|
|
|
const eventsAlreadyAdded = await indexManager.addHistoricEvents(
|
2019-11-12 15:29:07 +03:00
|
|
|
events, newCheckpoint, checkpoint);
|
2020-05-20 12:01:56 +03:00
|
|
|
|
|
|
|
// We didn't get a valid new checkpoint from the server, nothing
|
|
|
|
// to do here anymore.
|
|
|
|
if (!newCheckpoint) {
|
|
|
|
console.log("EventIndex: The server didn't return a valid ",
|
2021-04-27 18:23:27 +03:00
|
|
|
"new checkpoint, not continuing the crawl.", checkpoint);
|
2020-05-20 12:14:37 +03:00
|
|
|
continue;
|
2020-05-20 12:01:56 +03:00
|
|
|
}
|
|
|
|
|
2019-11-12 15:29:07 +03:00
|
|
|
// If all events were already indexed we assume that we catched
|
|
|
|
// up with our index and don't need to crawl the room further.
|
|
|
|
// Let us delete the checkpoint in that case, otherwise push
|
|
|
|
// the new checkpoint to be used by the crawler.
|
|
|
|
if (eventsAlreadyAdded === true && newCheckpoint.fullCrawl !== true) {
|
2019-11-13 17:26:27 +03:00
|
|
|
console.log("EventIndex: Checkpoint had already all events",
|
2021-04-27 18:23:27 +03:00
|
|
|
"added, stopping the crawl", checkpoint);
|
2019-11-13 14:25:16 +03:00
|
|
|
await indexManager.removeCrawlerCheckpoint(newCheckpoint);
|
2019-11-12 15:29:07 +03:00
|
|
|
} else {
|
2020-04-08 12:15:13 +03:00
|
|
|
if (eventsAlreadyAdded === true) {
|
|
|
|
console.log("EventIndex: Checkpoint had already all events",
|
2021-04-27 18:23:27 +03:00
|
|
|
"added, but continuing due to a full crawl", checkpoint);
|
2020-04-08 12:15:13 +03:00
|
|
|
}
|
2019-11-13 17:57:12 +03:00
|
|
|
this.crawlerCheckpoints.push(newCheckpoint);
|
2019-11-12 15:29:07 +03:00
|
|
|
}
|
|
|
|
} catch (e) {
|
2019-11-13 17:26:27 +03:00
|
|
|
console.log("EventIndex: Error durring a crawl", e);
|
2019-11-18 17:04:44 +03:00
|
|
|
// An error occurred, put the checkpoint back so we
|
2019-11-12 15:29:07 +03:00
|
|
|
// can retry.
|
2019-11-13 17:57:12 +03:00
|
|
|
this.crawlerCheckpoints.push(checkpoint);
|
2019-11-12 15:29:07 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-11-18 17:04:22 +03:00
|
|
|
this._crawler = null;
|
2019-11-12 15:29:07 +03:00
|
|
|
}
|
|
|
|
|
2020-01-28 17:06:43 +03:00
|
|
|
/**
|
|
|
|
* Start the crawler background task.
|
|
|
|
*/
|
2019-11-12 15:29:07 +03:00
|
|
|
startCrawler() {
|
2019-11-18 17:04:22 +03:00
|
|
|
if (this._crawler !== null) return;
|
|
|
|
this.crawlerFunc();
|
2019-11-12 15:29:07 +03:00
|
|
|
}
|
|
|
|
|
2020-01-28 17:06:43 +03:00
|
|
|
/**
|
|
|
|
* Stop the crawler background task.
|
|
|
|
*/
|
2019-11-14 18:13:22 +03:00
|
|
|
stopCrawler() {
|
2019-11-18 17:04:22 +03:00
|
|
|
if (this._crawler === null) return;
|
|
|
|
this._crawler.cancel();
|
2019-11-12 15:29:07 +03:00
|
|
|
}
|
2019-11-12 17:39:26 +03:00
|
|
|
|
2020-01-28 17:06:43 +03:00
|
|
|
/**
|
|
|
|
* Close the event index.
|
|
|
|
*
|
|
|
|
* This removes all the MatrixClient event listeners, stops the crawler
|
|
|
|
* task, and closes the index.
|
|
|
|
*/
|
2019-11-14 18:13:22 +03:00
|
|
|
async close() {
|
|
|
|
const indexManager = PlatformPeg.get().getEventIndexingManager();
|
2019-11-19 12:46:18 +03:00
|
|
|
this.removeListeners();
|
2019-11-14 18:13:22 +03:00
|
|
|
this.stopCrawler();
|
2020-03-13 13:33:58 +03:00
|
|
|
await indexManager.closeEventIndex();
|
|
|
|
return;
|
2019-11-14 18:13:22 +03:00
|
|
|
}
|
|
|
|
|
2020-01-28 17:06:43 +03:00
|
|
|
/**
|
|
|
|
* Search the event index using the given term for matching events.
|
|
|
|
*
|
|
|
|
* @param {SearchArgs} searchArgs The search configuration for the search,
|
|
|
|
* sets the search term and determines the search result contents.
|
|
|
|
*
|
|
|
|
* @return {Promise<[SearchResult]>} A promise that will resolve to an array
|
|
|
|
* of search results once the search is done.
|
|
|
|
*/
|
2019-11-12 17:39:26 +03:00
|
|
|
async search(searchArgs) {
|
2019-11-13 14:25:16 +03:00
|
|
|
const indexManager = PlatformPeg.get().getEventIndexingManager();
|
|
|
|
return indexManager.searchEventIndex(searchArgs);
|
2019-11-12 17:39:26 +03:00
|
|
|
}
|
2020-01-15 14:05:02 +03:00
|
|
|
|
2020-01-24 16:24:25 +03:00
|
|
|
/**
|
|
|
|
* Load events that contain URLs from the event index.
|
|
|
|
*
|
|
|
|
* @param {Room} room The room for which we should fetch events containing
|
|
|
|
* URLs
|
|
|
|
*
|
|
|
|
* @param {number} limit The maximum number of events to fetch.
|
|
|
|
*
|
|
|
|
* @param {string} fromEvent From which event should we continue fetching
|
|
|
|
* events from the index. This is only needed if we're continuing to fill
|
|
|
|
* the timeline, e.g. if we're paginating. This needs to be set to a event
|
|
|
|
* id of an event that was previously fetched with this function.
|
|
|
|
*
|
|
|
|
* @param {string} direction The direction in which we will continue
|
|
|
|
* fetching events. EventTimeline.BACKWARDS to continue fetching events that
|
|
|
|
* are older than the event given in fromEvent, EventTimeline.FORWARDS to
|
|
|
|
* fetch newer events.
|
|
|
|
*
|
|
|
|
* @returns {Promise<MatrixEvent[]>} Resolves to an array of events that
|
|
|
|
* contain URLs.
|
|
|
|
*/
|
2020-01-17 12:04:53 +03:00
|
|
|
async loadFileEvents(room, limit = 10, fromEvent = null, direction = EventTimeline.BACKWARDS) {
|
2020-01-15 14:05:02 +03:00
|
|
|
const client = MatrixClientPeg.get();
|
|
|
|
const indexManager = PlatformPeg.get().getEventIndexingManager();
|
|
|
|
|
2020-01-17 13:52:20 +03:00
|
|
|
const loadArgs = {
|
2020-01-17 12:04:53 +03:00
|
|
|
roomId: room.roomId,
|
2020-01-17 13:52:20 +03:00
|
|
|
limit: limit,
|
|
|
|
};
|
2020-01-17 12:04:53 +03:00
|
|
|
|
|
|
|
if (fromEvent) {
|
|
|
|
loadArgs.fromEvent = fromEvent;
|
|
|
|
loadArgs.direction = direction;
|
|
|
|
}
|
|
|
|
|
2020-01-17 13:52:20 +03:00
|
|
|
let events;
|
2020-01-17 12:04:53 +03:00
|
|
|
|
2020-01-15 14:05:02 +03:00
|
|
|
// Get our events from the event index.
|
2020-01-17 12:04:53 +03:00
|
|
|
try {
|
|
|
|
events = await indexManager.loadFileEvents(loadArgs);
|
|
|
|
} catch (e) {
|
|
|
|
console.log("EventIndex: Error getting file events", e);
|
2020-01-17 13:52:20 +03:00
|
|
|
return [];
|
2020-01-17 12:04:53 +03:00
|
|
|
}
|
2020-01-15 14:05:02 +03:00
|
|
|
|
2020-01-17 13:52:20 +03:00
|
|
|
const eventMapper = client.getEventMapper();
|
2020-01-15 14:05:02 +03:00
|
|
|
|
|
|
|
// Turn the events into MatrixEvent objects.
|
|
|
|
const matrixEvents = events.map(e => {
|
|
|
|
const matrixEvent = eventMapper(e.event);
|
|
|
|
|
2020-01-20 14:41:55 +03:00
|
|
|
const member = new RoomMember(room.roomId, matrixEvent.getSender());
|
2020-01-15 14:05:02 +03:00
|
|
|
|
|
|
|
// We can't really reconstruct the whole room state from our
|
|
|
|
// EventIndex to calculate the correct display name. Use the
|
|
|
|
// disambiguated form always instead.
|
|
|
|
member.name = e.profile.displayname + " (" + matrixEvent.getSender() + ")";
|
|
|
|
|
|
|
|
// This is sets the avatar URL.
|
|
|
|
const memberEvent = eventMapper(
|
|
|
|
{
|
|
|
|
content: {
|
|
|
|
membership: "join",
|
|
|
|
avatar_url: e.profile.avatar_url,
|
|
|
|
displayname: e.profile.displayname,
|
|
|
|
},
|
|
|
|
type: "m.room.member",
|
|
|
|
event_id: matrixEvent.getId() + ":eventIndex",
|
|
|
|
room_id: matrixEvent.getRoomId(),
|
|
|
|
sender: matrixEvent.getSender(),
|
|
|
|
origin_server_ts: matrixEvent.getTs(),
|
2020-01-17 13:52:20 +03:00
|
|
|
state_key: matrixEvent.getSender(),
|
|
|
|
},
|
2020-01-15 14:05:02 +03:00
|
|
|
);
|
|
|
|
|
|
|
|
// We set this manually to avoid emitting RoomMember.membership and
|
|
|
|
// RoomMember.name events.
|
|
|
|
member.events.member = memberEvent;
|
|
|
|
matrixEvent.sender = member;
|
|
|
|
|
|
|
|
return matrixEvent;
|
|
|
|
});
|
|
|
|
|
2020-01-17 12:04:53 +03:00
|
|
|
return matrixEvents;
|
|
|
|
}
|
|
|
|
|
2020-01-24 16:24:25 +03:00
|
|
|
/**
|
|
|
|
* Fill a timeline with events that contain URLs.
|
|
|
|
*
|
|
|
|
* @param {TimelineSet} timelineSet The TimelineSet the Timeline belongs to,
|
|
|
|
* used to check if we're adding duplicate events.
|
|
|
|
*
|
|
|
|
* @param {Timeline} timeline The Timeline which should be filed with
|
|
|
|
* events.
|
|
|
|
*
|
|
|
|
* @param {Room} room The room for which we should fetch events containing
|
|
|
|
* URLs
|
|
|
|
*
|
|
|
|
* @param {number} limit The maximum number of events to fetch.
|
|
|
|
*
|
|
|
|
* @param {string} fromEvent From which event should we continue fetching
|
|
|
|
* events from the index. This is only needed if we're continuing to fill
|
|
|
|
* the timeline, e.g. if we're paginating. This needs to be set to a event
|
|
|
|
* id of an event that was previously fetched with this function.
|
|
|
|
*
|
|
|
|
* @param {string} direction The direction in which we will continue
|
|
|
|
* fetching events. EventTimeline.BACKWARDS to continue fetching events that
|
|
|
|
* are older than the event given in fromEvent, EventTimeline.FORWARDS to
|
|
|
|
* fetch newer events.
|
|
|
|
*
|
|
|
|
* @returns {Promise<boolean>} Resolves to true if events were added to the
|
|
|
|
* timeline, false otherwise.
|
|
|
|
*/
|
2021-04-29 20:57:02 +03:00
|
|
|
async populateFileTimeline(
|
|
|
|
timelineSet,
|
|
|
|
timeline,
|
|
|
|
room,
|
|
|
|
limit = 10,
|
|
|
|
fromEvent = null,
|
|
|
|
direction = EventTimeline.BACKWARDS,
|
|
|
|
) {
|
2020-01-17 13:52:20 +03:00
|
|
|
const matrixEvents = await this.loadFileEvents(room, limit, fromEvent, direction);
|
2020-01-17 12:04:53 +03:00
|
|
|
|
2020-01-22 18:11:54 +03:00
|
|
|
// If this is a normal fill request, not a pagination request, we need
|
|
|
|
// to get our events in the BACKWARDS direction but populate them in the
|
|
|
|
// forwards direction.
|
|
|
|
// This needs to happen because a fill request might come with an
|
|
|
|
// exisitng timeline e.g. if you close and re-open the FilePanel.
|
|
|
|
if (fromEvent === null) {
|
|
|
|
matrixEvents.reverse();
|
|
|
|
direction = direction == EventTimeline.BACKWARDS ? EventTimeline.FORWARDS: EventTimeline.BACKWARDS;
|
|
|
|
}
|
|
|
|
|
2020-01-24 16:24:25 +03:00
|
|
|
// Add the events to the timeline of the file panel.
|
2020-01-15 14:05:02 +03:00
|
|
|
matrixEvents.forEach(e => {
|
|
|
|
if (!timelineSet.eventIdToTimeline(e.getId())) {
|
2020-01-20 14:43:20 +03:00
|
|
|
timelineSet.addEventToTimeline(e, timeline, direction == EventTimeline.BACKWARDS);
|
2020-01-15 14:05:02 +03:00
|
|
|
}
|
|
|
|
});
|
|
|
|
|
2020-03-20 15:57:29 +03:00
|
|
|
let ret = false;
|
|
|
|
let paginationToken = "";
|
|
|
|
|
2020-01-15 14:05:02 +03:00
|
|
|
// Set the pagination token to the oldest event that we retrieved.
|
2020-01-17 12:04:53 +03:00
|
|
|
if (matrixEvents.length > 0) {
|
2020-03-20 15:57:29 +03:00
|
|
|
paginationToken = matrixEvents[matrixEvents.length - 1].getId();
|
|
|
|
ret = true;
|
2020-01-17 12:04:53 +03:00
|
|
|
}
|
2020-03-20 15:57:29 +03:00
|
|
|
|
|
|
|
console.log("EventIndex: Populating file panel with", matrixEvents.length,
|
2021-04-27 18:23:27 +03:00
|
|
|
"events and setting the pagination token to", paginationToken);
|
2020-03-20 15:57:29 +03:00
|
|
|
|
|
|
|
timeline.setPaginationToken(paginationToken, EventTimeline.BACKWARDS);
|
|
|
|
return ret;
|
2020-01-17 12:04:53 +03:00
|
|
|
}
|
|
|
|
|
2020-01-24 16:24:25 +03:00
|
|
|
/**
|
|
|
|
* Emulate a TimelineWindow pagination() request with the event index as the event source
|
|
|
|
*
|
|
|
|
* Might not fetch events from the index if the timeline already contains
|
|
|
|
* events that the window isn't showing.
|
|
|
|
*
|
|
|
|
* @param {Room} room The room for which we should fetch events containing
|
|
|
|
* URLs
|
|
|
|
*
|
|
|
|
* @param {TimelineWindow} timelineWindow The timeline window that should be
|
|
|
|
* populated with new events.
|
|
|
|
*
|
|
|
|
* @param {string} direction The direction in which we should paginate.
|
|
|
|
* EventTimeline.BACKWARDS to paginate back, EventTimeline.FORWARDS to
|
|
|
|
* paginate forwards.
|
|
|
|
*
|
|
|
|
* @param {number} limit The maximum number of events to fetch while
|
|
|
|
* paginating.
|
|
|
|
*
|
|
|
|
* @returns {Promise<boolean>} Resolves to a boolean which is true if more
|
|
|
|
* events were successfully retrieved.
|
|
|
|
*/
|
2020-01-17 12:04:53 +03:00
|
|
|
paginateTimelineWindow(room, timelineWindow, direction, limit) {
|
2020-01-21 17:45:29 +03:00
|
|
|
const tl = timelineWindow.getTimelineIndex(direction);
|
2020-01-17 12:04:53 +03:00
|
|
|
|
|
|
|
if (!tl) return Promise.resolve(false);
|
|
|
|
if (tl.pendingPaginate) return tl.pendingPaginate;
|
|
|
|
|
2020-01-21 17:45:29 +03:00
|
|
|
if (timelineWindow.extend(direction, limit)) {
|
2020-01-17 12:04:53 +03:00
|
|
|
return Promise.resolve(true);
|
|
|
|
}
|
|
|
|
|
|
|
|
const paginationMethod = async (timelineWindow, timeline, room, direction, limit) => {
|
|
|
|
const timelineSet = timelineWindow._timelineSet;
|
|
|
|
const token = timeline.timeline.getPaginationToken(direction);
|
|
|
|
|
2020-01-20 14:43:20 +03:00
|
|
|
const ret = await this.populateFileTimeline(timelineSet, timeline.timeline, room, limit, token, direction);
|
2020-01-17 12:04:53 +03:00
|
|
|
|
|
|
|
timeline.pendingPaginate = null;
|
2020-01-21 17:45:29 +03:00
|
|
|
timelineWindow.extend(direction, limit);
|
2020-01-17 12:04:53 +03:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
};
|
|
|
|
|
2020-01-20 14:43:20 +03:00
|
|
|
const paginationPromise = paginationMethod(timelineWindow, tl, room, direction, limit);
|
2020-01-17 12:04:53 +03:00
|
|
|
tl.pendingPaginate = paginationPromise;
|
|
|
|
|
|
|
|
return paginationPromise;
|
2020-01-15 14:05:02 +03:00
|
|
|
}
|
2020-01-27 19:18:17 +03:00
|
|
|
|
2020-01-28 17:06:43 +03:00
|
|
|
/**
|
|
|
|
* Get statistical information of the index.
|
|
|
|
*
|
|
|
|
* @return {Promise<IndexStats>} A promise that will resolve to the index
|
|
|
|
* statistics.
|
|
|
|
*/
|
2020-01-20 19:42:24 +03:00
|
|
|
async getStats() {
|
2019-11-26 15:14:53 +03:00
|
|
|
const indexManager = PlatformPeg.get().getEventIndexingManager();
|
2020-01-20 19:42:24 +03:00
|
|
|
return indexManager.getStats();
|
2019-11-26 15:14:53 +03:00
|
|
|
}
|
2019-11-26 15:37:07 +03:00
|
|
|
|
2020-06-08 17:43:20 +03:00
|
|
|
/**
|
|
|
|
* Check if the room with the given id is already indexed.
|
|
|
|
*
|
|
|
|
* @param {string} roomId The ID of the room which we want to check if it
|
|
|
|
* has been already indexed.
|
|
|
|
*
|
|
|
|
* @return {Promise<boolean>} Returns true if the index contains events for
|
|
|
|
* the given room, false otherwise.
|
|
|
|
*/
|
|
|
|
async isRoomIndexed(roomId) {
|
|
|
|
const indexManager = PlatformPeg.get().getEventIndexingManager();
|
|
|
|
return indexManager.isRoomIndexed(roomId);
|
|
|
|
}
|
|
|
|
|
2020-01-20 19:42:24 +03:00
|
|
|
/**
|
|
|
|
* Get the room that we are currently crawling.
|
|
|
|
*
|
2020-01-21 18:40:32 +03:00
|
|
|
* @returns {Room} A MatrixRoom that is being currently crawled, null
|
|
|
|
* if no room is currently being crawled.
|
2020-01-20 19:42:24 +03:00
|
|
|
*/
|
|
|
|
currentRoom() {
|
|
|
|
if (this._currentCheckpoint === null && this.crawlerCheckpoints.length === 0) {
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
|
|
|
const client = MatrixClientPeg.get();
|
|
|
|
|
|
|
|
if (this._currentCheckpoint !== null) {
|
|
|
|
return client.getRoom(this._currentCheckpoint.roomId);
|
|
|
|
} else {
|
|
|
|
return client.getRoom(this.crawlerCheckpoints[0].roomId);
|
|
|
|
}
|
|
|
|
}
|
2020-02-01 14:00:04 +03:00
|
|
|
|
|
|
|
crawlingRooms() {
|
|
|
|
const totalRooms = new Set();
|
|
|
|
const crawlingRooms = new Set();
|
|
|
|
|
|
|
|
this.crawlerCheckpoints.forEach((checkpoint, index) => {
|
|
|
|
crawlingRooms.add(checkpoint.roomId);
|
|
|
|
});
|
|
|
|
|
|
|
|
if (this._currentCheckpoint !== null) {
|
|
|
|
crawlingRooms.add(this._currentCheckpoint.roomId);
|
|
|
|
}
|
|
|
|
|
|
|
|
const client = MatrixClientPeg.get();
|
|
|
|
const rooms = client.getRooms();
|
|
|
|
|
|
|
|
const isRoomEncrypted = (room) => {
|
|
|
|
return client.isRoomEncrypted(room.roomId);
|
|
|
|
};
|
|
|
|
|
|
|
|
const encryptedRooms = rooms.filter(isRoomEncrypted);
|
|
|
|
encryptedRooms.forEach((room, index) => {
|
|
|
|
totalRooms.add(room.roomId);
|
|
|
|
});
|
|
|
|
|
|
|
|
return {crawlingRooms, totalRooms};
|
|
|
|
}
|
2019-11-12 15:29:07 +03:00
|
|
|
}
|