From 3718826e94bc163250c0f5ed397f4dd41f4a3e92 Mon Sep 17 00:00:00 2001 From: Jaiwanth Date: Fri, 25 Jun 2021 11:16:59 +0530 Subject: [PATCH] refactor to share downloading code across all formats --- src/components/views/rooms/RoomHeader.js | 4 +- src/utils/exportUtils/Exporter.ts | 67 +++++++++++++++++++++--- src/utils/exportUtils/HtmlExport.tsx | 37 ++----------- src/utils/exportUtils/JSONExport.ts | 43 ++++----------- src/utils/exportUtils/PlainTextExport.ts | 44 +++------------- src/utils/exportUtils/StreamToZip.ts | 2 +- src/utils/exportUtils/exportUtils.ts | 4 +- 7 files changed, 83 insertions(+), 118 deletions(-) diff --git a/src/components/views/rooms/RoomHeader.js b/src/components/views/rooms/RoomHeader.js index f37145261c..d977b6d87b 100644 --- a/src/components/views/rooms/RoomHeader.js +++ b/src/components/views/rooms/RoomHeader.js @@ -84,11 +84,11 @@ export default class RoomHeader extends React.Component { _exportConversationalHistory = async () => { await exportConversationalHistory( this.props.room, - exportFormats.JSON, + exportFormats.PLAIN_TEXT, exportTypes.START_DATE, { startDate: parseInt(new Date("2021.05.20").getTime().toFixed(0)), - attachmentsIncluded: false, + attachmentsIncluded: true, maxSize: 7 * 1024 * 1024, // 7 MB }, ); diff --git a/src/utils/exportUtils/Exporter.ts b/src/utils/exportUtils/Exporter.ts index 5e766ff7ba..028b5c808e 100644 --- a/src/utils/exportUtils/Exporter.ts +++ b/src/utils/exportUtils/Exporter.ts @@ -1,3 +1,4 @@ +import streamSaver from "streamsaver"; import { MatrixEvent } from "matrix-js-sdk/src/models/event"; import { Room } from "matrix-js-sdk/src/models/room"; import { MatrixClientPeg } from "../../MatrixClientPeg"; @@ -6,6 +7,9 @@ import { decryptFile } from "../DecryptFile"; import { mediaFromContent } from "../../customisations/Media"; import { formatFullDateNoDay } from "../../DateUtils"; import { MatrixClient } from "matrix-js-sdk"; +import streamToZIP from "./StreamToZip"; +import * as ponyfill from "web-streams-polyfill/ponyfill" +import "web-streams-polyfill/ponyfill"; // to support streams API for older browsers type FileStream = { name: string, @@ -15,6 +19,9 @@ type FileStream = { export default abstract class Exporter { protected files: FileStream[]; protected client: MatrixClient; + protected writer: WritableStreamDefaultWriter; + protected fileStream: WritableStream; + protected constructor( protected room: Room, protected exportType: exportTypes, @@ -22,9 +29,16 @@ export default abstract class Exporter { ) { this.files = []; this.client = MatrixClientPeg.get(); + window.addEventListener("beforeunload", this.onBeforeUnload); + window.addEventListener("onunload", this.abortExport); } - protected addFile = (filePath: string, blob: Blob) => { + protected onBeforeUnload(e: BeforeUnloadEvent) { + e.preventDefault(); + return e.returnValue = "Are you sure you want to exit during this export?"; + } + + protected addFile(filePath: string, blob: Blob) { const file = { name: filePath, stream: () => blob.stream(), @@ -32,16 +46,53 @@ export default abstract class Exporter { this.files.push(file); } - protected pumpToFileStream = async (reader: ReadableStreamDefaultReader, writer: WritableStreamDefaultWriter) => { + protected async downloadZIP() { + const filename = `matrix-export-${formatFullDateNoDay(new Date())}.zip`; + //Support for firefox browser + streamSaver.WritableStream = ponyfill.WritableStream + //Create a writable stream to the directory + this.fileStream = streamSaver.createWriteStream(filename); + + this.writer = this.fileStream.getWriter(); + const files = this.files; + + console.info("Generating a ZIP..."); + const readableZipStream = streamToZIP({ + start(ctrl) { + for (const file of files) ctrl.enqueue(file); + ctrl.close(); + }, + }); + + console.info("Writing to the file system...") + + const reader = readableZipStream.getReader() + await this.pumpToFileStream(reader); + } + + protected async downloadPlainText(fileName: string, text: string): Promise { + this.fileStream = streamSaver.createWriteStream(fileName); + this.writer = this.fileStream.getWriter() + const data = new TextEncoder().encode(text); + await this.writer.write(data); + await this.writer.close(); + } + + protected async abortExport(): Promise { + if (this.fileStream) await this.fileStream.abort(); + if (this.writer) await this.writer.abort(); + } + + protected async pumpToFileStream(reader: ReadableStreamDefaultReader) { const res = await reader.read(); - if (res.done) await writer.close(); + if (res.done) await this.writer.close(); else { - await writer.write(res.value); - await this.pumpToFileStream(reader, writer) + await this.writer.write(res.value); + await this.pumpToFileStream(reader); } } - protected setEventMetadata = (event: MatrixEvent) => { + protected setEventMetadata(event: MatrixEvent) { const roomState = this.client.getRoom(this.room.roomId).currentState; event.sender = roomState.getSentinelMember( event.getSender(), @@ -54,7 +105,7 @@ export default abstract class Exporter { return event; } - protected getLimit = () => { + protected getLimit() { let limit: number; switch (this.exportType) { case exportTypes.LAST_N_MESSAGES: @@ -69,7 +120,7 @@ export default abstract class Exporter { return limit; } - protected getRequiredEvents = async () : Promise => { + protected async getRequiredEvents():Promise { const eventMapper = this.client.getEventMapper(); let prevToken: string|null = null; diff --git a/src/utils/exportUtils/HtmlExport.tsx b/src/utils/exportUtils/HtmlExport.tsx index 2db4e243a7..6415f996db 100644 --- a/src/utils/exportUtils/HtmlExport.tsx +++ b/src/utils/exportUtils/HtmlExport.tsx @@ -1,5 +1,4 @@ import React from "react" -import streamSaver from "streamsaver"; import Exporter from "./Exporter"; import { mediaFromMxc } from "../../customisations/Media"; import { Room } from "matrix-js-sdk/src/models/room"; @@ -7,11 +6,10 @@ import { MatrixEvent } from "matrix-js-sdk/src/models/event"; import { renderToStaticMarkup } from 'react-dom/server' import { Layout } from "../../settings/Layout"; import { shouldFormContinuation } from "../../components/structures/MessagePanel"; -import { formatFullDateNoDay, formatFullDateNoDayNoTime, wantsDateSeparator } from "../../DateUtils"; +import { formatFullDateNoDayNoTime, wantsDateSeparator } from "../../DateUtils"; import { RoomPermalinkCreator } from "../permalinks/Permalinks"; import { _t } from "../../languageHandler"; import { EventType } from "matrix-js-sdk/src/@types/event"; -import * as ponyfill from "web-streams-polyfill/ponyfill" import * as Avatar from "../../Avatar"; import EventTile, { haveTileForEvent } from "../../components/views/rooms/EventTile"; import DateSeparator from "../../components/views/messages/DateSeparator"; @@ -22,7 +20,6 @@ import exportIcons from "./exportIcons"; import { exportTypes } from "./exportUtils"; import { exportOptions } from "./exportUtils"; import MatrixClientContext from "../../contexts/MatrixClientContext"; -import zip from "./StreamToZip"; export default class HTMLExporter extends Exporter { protected avatars: Map; @@ -38,12 +35,6 @@ export default class HTMLExporter extends Exporter { this.mediaOmitText = !this.exportOptions.attachmentsIncluded ? _t("Media omitted") : _t("Media omitted - file size limit exceeded"); - window.addEventListener("beforeunload", this.onBeforeUnload) - } - - protected onBeforeUnload = (e: BeforeUnloadEvent) => { - e.preventDefault(); - return e.returnValue = "Are you sure you want to exit during this export?"; } protected async getRoomAvatar() { @@ -267,7 +258,7 @@ export default class HTMLExporter extends Exporter { return eventTileMarkup; } - protected createModifiedEvent = (text: string, mxEv: MatrixEvent) => { + protected createModifiedEvent(text: string, mxEv: MatrixEvent) { const modifiedContent = { msgtype: "m.text", body: `*${text}*`, @@ -351,34 +342,14 @@ export default class HTMLExporter extends Exporter { this.addFile(`icons/${iconName}`, new Blob([exportIcons[iconName]])); } - const filename = `matrix-export-${formatFullDateNoDay(new Date())}.zip`; - console.info("HTML creation successful!"); - //Support for firefox browser - streamSaver.WritableStream = ponyfill.WritableStream - //Create a writable stream to the directory - const fileStream = streamSaver.createWriteStream(filename); - - const writer = fileStream.getWriter(); - const files = this.files; - - console.info("Generating a ZIP..."); - const readableZipStream = zip({ - start(ctrl) { - for (const file of files) ctrl.enqueue(file); - ctrl.close(); - }, - }); - - console.info("Writing to file system...") - - const reader = readableZipStream.getReader() - await this.pumpToFileStream(reader, writer); + await this.downloadZIP(); const exportEnd = performance.now(); console.info(`Export Successful! Exported ${res.length} events in ${(exportEnd - fetchStart)/1000} seconds`); window.removeEventListener("beforeunload", this.onBeforeUnload); + window.removeEventListener("onunload", this.abortExport); } } diff --git a/src/utils/exportUtils/JSONExport.ts b/src/utils/exportUtils/JSONExport.ts index af62428ae1..b261b305f9 100644 --- a/src/utils/exportUtils/JSONExport.ts +++ b/src/utils/exportUtils/JSONExport.ts @@ -1,13 +1,10 @@ -import streamSaver from "streamsaver"; import Exporter from "./Exporter"; import { Room } from "matrix-js-sdk/src/models/room"; import { MatrixEvent } from "matrix-js-sdk/src/models/event"; import { formatFullDateNoDay, formatFullDateNoDayNoTime } from "../../DateUtils"; -import * as ponyfill from "web-streams-polyfill/ponyfill" import { haveTileForEvent } from "../../components/views/rooms/EventTile"; import { exportTypes } from "./exportUtils"; import { exportOptions } from "./exportUtils"; -import zip from "./StreamToZip"; import { EventType } from "matrix-js-sdk/src/@types/event"; @@ -17,7 +14,6 @@ export default class JSONExporter extends Exporter { constructor(room: Room, exportType: exportTypes, exportOptions: exportOptions) { super(room, exportType, exportOptions); this.totalSize = 0; - window.addEventListener("beforeunload", this.onBeforeUnload) } protected wrapJSON(json: string): string { @@ -39,15 +35,10 @@ ${json} }` } - protected indentEachLine(string: string) { + protected indentEachLine(JSONString: string, spaces: number) { const indent = ' '; const regex = /^(?!\s*$)/gm; - return string.replace(regex, indent.repeat(2)); - } - - protected onBeforeUnload = (e: BeforeUnloadEvent) => { - e.preventDefault(); - return e.returnValue = "Are you sure you want to exit during this export?"; + return JSONString.replace(regex, indent.repeat(spaces)); } protected async getJSONString(mxEv: MatrixEvent) { @@ -76,7 +67,7 @@ ${json} if (!haveTileForEvent(event)) continue; content += await this.getJSONString(event); } - return this.wrapJSON(this.indentEachLine(content.slice(0, -2))); + return this.wrapJSON(this.indentEachLine(content.slice(0, -2), 2)); } public async export() { @@ -91,34 +82,18 @@ ${json} const text = await this.createOutput(res); - console.info("Writing to the file system..."); - streamSaver.WritableStream = ponyfill.WritableStream - - const fileName = `matrix-export-${formatFullDateNoDay(new Date())}.json`; - const files = this.files; - if (files.length) { - this.addFile("result.json", new Blob([text])); - const fileStream = streamSaver.createWriteStream(fileName.slice(0, -5) + ".zip"); - const readableZipStream = zip({ - start(ctrl) { - for (const file of files) ctrl.enqueue(file); - ctrl.close(); - }, - }); - const writer = fileStream.getWriter() - const reader = readableZipStream.getReader() - await this.pumpToFileStream(reader, writer); + if (this.files.length) { + this.addFile("export.json", new Blob([text])); + await this.downloadZIP(); } else { - const fileStream = streamSaver.createWriteStream(fileName); - const writer = fileStream.getWriter() - const data = new TextEncoder().encode(text); - await writer.write(data); - await writer.close(); + const fileName = `matrix-export-${formatFullDateNoDay(new Date())}.json`; + await this.downloadPlainText(fileName, text); } const exportEnd = performance.now(); console.info(`Export Successful! Exported ${res.length} events in ${(exportEnd - fetchStart)/1000} seconds`); window.removeEventListener("beforeunload", this.onBeforeUnload); + window.removeEventListener("onunload", this.abortExport); } } diff --git a/src/utils/exportUtils/PlainTextExport.ts b/src/utils/exportUtils/PlainTextExport.ts index 89a9ff4b98..61665d4646 100644 --- a/src/utils/exportUtils/PlainTextExport.ts +++ b/src/utils/exportUtils/PlainTextExport.ts @@ -1,35 +1,24 @@ -import streamSaver from "streamsaver"; import Exporter from "./Exporter"; import { Room } from "matrix-js-sdk/src/models/room"; import { MatrixEvent } from "matrix-js-sdk/src/models/event"; import { formatFullDateNoDay } from "../../DateUtils"; import { _t } from "../../languageHandler"; -import * as ponyfill from "web-streams-polyfill/ponyfill" import { haveTileForEvent } from "../../components/views/rooms/EventTile"; import { exportTypes } from "./exportUtils"; import { exportOptions } from "./exportUtils"; import { textForEvent } from "../../TextForEvent"; -import zip from "./StreamToZip"; export default class PlainTextExporter extends Exporter { protected totalSize: number; protected mediaOmitText: string; - private readonly fileDir: string; constructor(room: Room, exportType: exportTypes, exportOptions: exportOptions) { super(room, exportType, exportOptions); this.totalSize = 0; - this.fileDir = `matrix-export-${formatFullDateNoDay(new Date())}`; this.mediaOmitText = !this.exportOptions.attachmentsIncluded ? _t("Media omitted") : _t("Media omitted - file size limit exceeded"); - window.addEventListener("beforeunload", this.onBeforeUnload) - } - - protected onBeforeUnload = (e: BeforeUnloadEvent) => { - e.preventDefault(); - return e.returnValue = "Are you sure you want to exit during this export?"; } protected textForReplyEvent = (ev : MatrixEvent) => { @@ -90,12 +79,6 @@ export default class PlainTextExporter extends Exporter { return content; } - protected getFileName = () => { - if (this.exportOptions.attachmentsIncluded) { - return `${this.room.name}.txt`; - } else return `${this.fileDir}.txt` - } - public async export() { console.info("Starting export process..."); console.info("Fetching events..."); @@ -108,32 +91,17 @@ export default class PlainTextExporter extends Exporter { const text = await this.createOutput(res); - console.info("Writing to the file system..."); - streamSaver.WritableStream = ponyfill.WritableStream - - const files = this.files; - if (files.length) { - this.addFile(this.getFileName(), new Blob([text])); - const fileStream = streamSaver.createWriteStream(`${this.fileDir}.zip`); - const readableZipStream = zip({ - start(ctrl) { - for (const file of files) ctrl.enqueue(file); - ctrl.close(); - }, - }); - const writer = fileStream.getWriter() - const reader = readableZipStream.getReader() - await this.pumpToFileStream(reader, writer); + if (this.files.length) { + this.addFile("export.txt", new Blob([text])); + await this.downloadZIP(); } else { - const fileStream = streamSaver.createWriteStream(`${this.fileDir}.txt`); - const writer = fileStream.getWriter() - const data = new TextEncoder().encode(text); - await writer.write(data); - await writer.close(); + const fileName = `matrix-export-${formatFullDateNoDay(new Date())}.txt`; + await this.downloadPlainText(fileName, text); } const exportEnd = performance.now(); console.info(`Export Successful! Exported ${res.length} events in ${(exportEnd - fetchStart)/1000} seconds`); + window.removeEventListener("onunload", this.abortExport); window.removeEventListener("beforeunload", this.onBeforeUnload); } } diff --git a/src/utils/exportUtils/StreamToZip.ts b/src/utils/exportUtils/StreamToZip.ts index b98e3e142f..a411d35190 100644 --- a/src/utils/exportUtils/StreamToZip.ts +++ b/src/utils/exportUtils/StreamToZip.ts @@ -101,7 +101,7 @@ const pump = (zipObj: ZipObj) => zipObj.reader ? zipObj.reader.read().then(chunk } }) : undefined; -export default function ZIP(underlyingSource: UnderlyingSource) { +export default function streamToZIP(underlyingSource: UnderlyingSource) { const files = Object.create(null); const filenames: string[] = []; const encoder = new TextEncoder(); diff --git a/src/utils/exportUtils/exportUtils.ts b/src/utils/exportUtils/exportUtils.ts index ba87dbdd67..0439d51ee2 100644 --- a/src/utils/exportUtils/exportUtils.ts +++ b/src/utils/exportUtils/exportUtils.ts @@ -6,7 +6,7 @@ import PlainTextExporter from "./PlainTextExport"; export enum exportFormats { HTML = "HTML", JSON = "JSON", - LOGS = "LOGS", + PLAIN_TEXT = "PLAIN_TEXT", } export enum exportTypes { @@ -36,7 +36,7 @@ const exportConversationalHistory = async ( case exportFormats.JSON: await new JSONExporter(room, exportType, exportOptions).export(); break; - case exportFormats.LOGS: + case exportFormats.PLAIN_TEXT: await new PlainTextExporter(room, exportType, exportOptions).export(); break; }