From 112dfa2b96fc7e4d6dd10348ea293034eeca6b36 Mon Sep 17 00:00:00 2001 From: Jaiwanth Date: Thu, 24 Jun 2021 18:19:12 +0530 Subject: [PATCH] Finish JSON export implementation --- src/components/views/rooms/RoomHeader.js | 6 +- src/utils/exportUtils/Exporter.ts | 13 +-- src/utils/exportUtils/HtmlExport.tsx | 8 +- src/utils/exportUtils/JSONExport.ts | 124 +++++++++++++++++++++++ src/utils/exportUtils/PlainTextExport.ts | 1 - src/utils/exportUtils/exportUtils.ts | 2 + 6 files changed, 138 insertions(+), 16 deletions(-) create mode 100644 src/utils/exportUtils/JSONExport.ts diff --git a/src/components/views/rooms/RoomHeader.js b/src/components/views/rooms/RoomHeader.js index 891b690f7b..f37145261c 100644 --- a/src/components/views/rooms/RoomHeader.js +++ b/src/components/views/rooms/RoomHeader.js @@ -84,12 +84,12 @@ export default class RoomHeader extends React.Component { _exportConversationalHistory = async () => { await exportConversationalHistory( this.props.room, - exportFormats.HTML, + exportFormats.JSON, exportTypes.START_DATE, { startDate: parseInt(new Date("2021.05.20").getTime().toFixed(0)), - attachmentsIncluded: true, - maxSize: 7 * 1024 * 1024, // 3 MB + attachmentsIncluded: false, + maxSize: 7 * 1024 * 1024, // 7 MB }, ); } diff --git a/src/utils/exportUtils/Exporter.ts b/src/utils/exportUtils/Exporter.ts index f6dce9c8e8..5e766ff7ba 100644 --- a/src/utils/exportUtils/Exporter.ts +++ b/src/utils/exportUtils/Exporter.ts @@ -5,6 +5,7 @@ import { exportOptions, exportTypes } from "./exportUtils"; import { decryptFile } from "../DecryptFile"; import { mediaFromContent } from "../../customisations/Media"; import { formatFullDateNoDay } from "../../DateUtils"; +import { MatrixClient } from "matrix-js-sdk"; type FileStream = { name: string, @@ -13,12 +14,14 @@ type FileStream = { export default abstract class Exporter { protected files: FileStream[]; + protected client: MatrixClient; protected constructor( protected room: Room, protected exportType: exportTypes, protected exportOptions?: exportOptions, ) { this.files = []; + this.client = MatrixClientPeg.get(); } protected addFile = (filePath: string, blob: Blob) => { @@ -39,8 +42,7 @@ export default abstract class Exporter { } protected setEventMetadata = (event: MatrixEvent) => { - const client = MatrixClientPeg.get(); - const roomState = client.getRoom(this.room.roomId).currentState; + const roomState = this.client.getRoom(this.room.roomId).currentState; event.sender = roomState.getSentinelMember( event.getSender(), ); @@ -68,8 +70,7 @@ export default abstract class Exporter { } protected getRequiredEvents = async () : Promise => { - const client = MatrixClientPeg.get(); - const eventMapper = client.getEventMapper(); + const eventMapper = this.client.getEventMapper(); let prevToken: string|null = null; let limit = this.getLimit(); @@ -77,7 +78,7 @@ export default abstract class Exporter { while (limit) { const eventsPerCrawl = Math.min(limit, 1000); - const res: any = await client.createMessagesRequest(this.room.roomId, prevToken, eventsPerCrawl, "b"); + const res: any = await this.client.createMessagesRequest(this.room.roomId, prevToken, eventsPerCrawl, "b"); if (res.chunk.length === 0) break; @@ -102,7 +103,7 @@ export default abstract class Exporter { const decryptionPromises = events .filter(event => event.isEncrypted()) .map(event => { - return client.decryptEventIfNeeded(event, { + return this.client.decryptEventIfNeeded(event, { isRetry: true, emit: false, }); diff --git a/src/utils/exportUtils/HtmlExport.tsx b/src/utils/exportUtils/HtmlExport.tsx index e05a7bd494..2db4e243a7 100644 --- a/src/utils/exportUtils/HtmlExport.tsx +++ b/src/utils/exportUtils/HtmlExport.tsx @@ -10,7 +10,6 @@ import { shouldFormContinuation } from "../../components/structures/MessagePanel import { formatFullDateNoDay, formatFullDateNoDayNoTime, wantsDateSeparator } from "../../DateUtils"; import { RoomPermalinkCreator } from "../permalinks/Permalinks"; import { _t } from "../../languageHandler"; -import { MatrixClientPeg } from "../../MatrixClientPeg"; import { EventType } from "matrix-js-sdk/src/@types/event"; import * as ponyfill from "web-streams-polyfill/ponyfill" import * as Avatar from "../../Avatar"; @@ -23,20 +22,17 @@ import exportIcons from "./exportIcons"; import { exportTypes } from "./exportUtils"; import { exportOptions } from "./exportUtils"; import MatrixClientContext from "../../contexts/MatrixClientContext"; -import { MatrixClient } from "matrix-js-sdk"; import zip from "./StreamToZip"; export default class HTMLExporter extends Exporter { protected avatars: Map; protected permalinkCreator: RoomPermalinkCreator; - protected matrixClient: MatrixClient; protected totalSize: number; protected mediaOmitText: string; constructor(room: Room, exportType: exportTypes, exportOptions: exportOptions) { super(room, exportType, exportOptions); this.avatars = new Map(); - this.matrixClient = MatrixClientPeg.get(); this.permalinkCreator = new RoomPermalinkCreator(this.room); this.totalSize = 0; this.mediaOmitText = !this.exportOptions.attachmentsIncluded @@ -78,7 +74,7 @@ export default class HTMLExporter extends Exporter { const exportDate = formatFullDateNoDayNoTime(new Date()); const creator = this.room.currentState.getStateEvents(EventType.RoomCreate, "")?.getSender(); const creatorName = this.room?.getMember(creator)?.rawDisplayName || creator; - const exporter = this.matrixClient.getUserId(); + const exporter = this.client.getUserId(); const exporterName = this.room?.getMember(exporter)?.rawDisplayName; const topic = this.room.currentState.getStateEvents(EventType.RoomTopic, "")?.getContent()?.topic || ""; const createdText = _t("%(creatorName)s created this room.", { @@ -235,7 +231,7 @@ export default class HTMLExporter extends Exporter { if (hasAvatar) await this.saveAvatarIfNeeded(mxEv); const eventTile =
- + { + e.preventDefault(); + return e.returnValue = "Are you sure you want to exit during this export?"; + } + + protected async getJSONString(mxEv: MatrixEvent) { + if (this.exportOptions.attachmentsIncluded && this.isAttachment(mxEv)) { + try { + const blob = await this.getMediaBlob(mxEv); + this.totalSize += blob.size; + const filePath = this.getFilePath(mxEv); + this.addFile(filePath, blob); + if (this.totalSize > this.exportOptions.maxSize - 1024 * 1024) { + this.exportOptions.attachmentsIncluded = false; + } + } catch (err) { + console.log("Error fetching file: " + err); + } + } + const jsonEvent: any = mxEv.toJSON(); + const clearEvent = mxEv.isEncrypted() ? jsonEvent.decrypted : jsonEvent; + const jsonString = JSON.stringify(clearEvent, null, 2); + return jsonString.length > 2 ? jsonString + ",\n" : ""; + } + + protected async createOutput(events: MatrixEvent[]) { + let content = ""; + for (const event of events) { + if (!haveTileForEvent(event)) continue; + content += await this.getJSONString(event); + } + return this.wrapJSON(this.indentEachLine(content.slice(0, -2))); + } + + public async export() { + console.info("Starting export process..."); + console.info("Fetching events..."); + const fetchStart = performance.now(); + const res = await this.getRequiredEvents(); + const fetchEnd = performance.now(); + + console.log(`Fetched ${res.length} events in ${(fetchEnd - fetchStart)/1000} s`); + console.info("Creating Output..."); + + const text = await this.createOutput(res); + + console.info("Writing to the file system..."); + streamSaver.WritableStream = ponyfill.WritableStream + + const fileName = `matrix-export-${formatFullDateNoDay(new Date())}.json`; + const files = this.files; + if (files.length) { + this.addFile("result.json", new Blob([text])); + const fileStream = streamSaver.createWriteStream(fileName.slice(0, -5) + ".zip"); + const readableZipStream = zip({ + start(ctrl) { + for (const file of files) ctrl.enqueue(file); + ctrl.close(); + }, + }); + const writer = fileStream.getWriter() + const reader = readableZipStream.getReader() + await this.pumpToFileStream(reader, writer); + } else { + const fileStream = streamSaver.createWriteStream(fileName); + const writer = fileStream.getWriter() + const data = new TextEncoder().encode(text); + await writer.write(data); + await writer.close(); + } + + const exportEnd = performance.now(); + console.info(`Export Successful! Exported ${res.length} events in ${(exportEnd - fetchStart)/1000} seconds`); + window.removeEventListener("beforeunload", this.onBeforeUnload); + } +} + diff --git a/src/utils/exportUtils/PlainTextExport.ts b/src/utils/exportUtils/PlainTextExport.ts index 862fac3aa1..89a9ff4b98 100644 --- a/src/utils/exportUtils/PlainTextExport.ts +++ b/src/utils/exportUtils/PlainTextExport.ts @@ -5,7 +5,6 @@ import { MatrixEvent } from "matrix-js-sdk/src/models/event"; import { formatFullDateNoDay } from "../../DateUtils"; import { _t } from "../../languageHandler"; import * as ponyfill from "web-streams-polyfill/ponyfill" -import "web-streams-polyfill/ponyfill" // to support blob.stream() import { haveTileForEvent } from "../../components/views/rooms/EventTile"; import { exportTypes } from "./exportUtils"; import { exportOptions } from "./exportUtils"; diff --git a/src/utils/exportUtils/exportUtils.ts b/src/utils/exportUtils/exportUtils.ts index 22457f926b..ba87dbdd67 100644 --- a/src/utils/exportUtils/exportUtils.ts +++ b/src/utils/exportUtils/exportUtils.ts @@ -1,5 +1,6 @@ import { Room } from "matrix-js-sdk/src/models/room"; import HTMLExporter from "./HtmlExport"; +import JSONExporter from "./JSONExport"; import PlainTextExporter from "./PlainTextExport"; export enum exportFormats { @@ -33,6 +34,7 @@ const exportConversationalHistory = async ( await new HTMLExporter(room, exportType, exportOptions).export(); break; case exportFormats.JSON: + await new JSONExporter(room, exportType, exportOptions).export(); break; case exportFormats.LOGS: await new PlainTextExporter(room, exportType, exportOptions).export();