mirrors/element-web
1
0
Fork 0
mirror of https://github.com/element-hq/element-web.git synced 2024-12-14 02:07:29 +03:00
Code Issues Projects Releases Packages Wiki Activity

Finish JSON export implementation

Browse source
This commit is contained in:
Jaiwanth 2021-06-24 18:19:12 +05:30
parent 4bfac11911
commit 112dfa2b96
6 changed files with 138 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/components/views/rooms/RoomHeader.js
View file

@ -84,12 +84,12 @@ export default class RoomHeader extends React.Component {
_exportConversationalHistory = async () => {
await exportConversationalHistory(
this.props.room,
exportFormats.HTML,
exportFormats.JSON,
exportTypes.START_DATE,
{
startDate: parseInt(new Date("2021.05.20").getTime().toFixed(0)),
attachmentsIncluded: true,
maxSize: 7 * 1024 * 1024, // 3 MB
attachmentsIncluded: false,
maxSize: 7 * 1024 * 1024, // 7 MB
},
);
}

13
src/utils/exportUtils/Exporter.ts
View file

@ -5,6 +5,7 @@ import { exportOptions, exportTypes } from "./exportUtils";
import { decryptFile } from "../DecryptFile";
import { mediaFromContent } from "../../customisations/Media";
import { formatFullDateNoDay } from "../../DateUtils";
import { MatrixClient } from "matrix-js-sdk";
type FileStream = {
name: string,
@ -13,12 +14,14 @@ type FileStream = {
export default abstract class Exporter {
protected files: FileStream[];
protected client: MatrixClient;
protected constructor(
protected room: Room,
protected exportType: exportTypes,
protected exportOptions?: exportOptions,
) {
this.files = [];
this.client = MatrixClientPeg.get();
}
protected addFile = (filePath: string, blob: Blob) => {
@ -39,8 +42,7 @@ export default abstract class Exporter {
}
protected setEventMetadata = (event: MatrixEvent) => {
const client = MatrixClientPeg.get();
const roomState = client.getRoom(this.room.roomId).currentState;
const roomState = this.client.getRoom(this.room.roomId).currentState;
event.sender = roomState.getSentinelMember(
event.getSender(),
);
@ -68,8 +70,7 @@ export default abstract class Exporter {
}
protected getRequiredEvents = async () : Promise<MatrixEvent[]> => {
const client = MatrixClientPeg.get();
const eventMapper = client.getEventMapper();
const eventMapper = this.client.getEventMapper();
let prevToken: string|null = null;
let limit = this.getLimit();
@ -77,7 +78,7 @@ export default abstract class Exporter {
while (limit) {
const eventsPerCrawl = Math.min(limit, 1000);
const res: any = await client.createMessagesRequest(this.room.roomId, prevToken, eventsPerCrawl, "b");
const res: any = await this.client.createMessagesRequest(this.room.roomId, prevToken, eventsPerCrawl, "b");
if (res.chunk.length === 0) break;
@ -102,7 +103,7 @@ export default abstract class Exporter {
const decryptionPromises = events
.filter(event => event.isEncrypted())
.map(event => {
return client.decryptEventIfNeeded(event, {
return this.client.decryptEventIfNeeded(event, {
isRetry: true,
emit: false,
});

8
src/utils/exportUtils/HtmlExport.tsx
View file

@ -10,7 +10,6 @@ import { shouldFormContinuation } from "../../components/structures/MessagePanel
import { formatFullDateNoDay, formatFullDateNoDayNoTime, wantsDateSeparator } from "../../DateUtils";
import { RoomPermalinkCreator } from "../permalinks/Permalinks";
import { _t } from "../../languageHandler";
import { MatrixClientPeg } from "../../MatrixClientPeg";
import { EventType } from "matrix-js-sdk/src/@types/event";
import * as ponyfill from "web-streams-polyfill/ponyfill"
import * as Avatar from "../../Avatar";
@ -23,20 +22,17 @@ import exportIcons from "./exportIcons";
import { exportTypes } from "./exportUtils";
import { exportOptions } from "./exportUtils";
import MatrixClientContext from "../../contexts/MatrixClientContext";
import { MatrixClient } from "matrix-js-sdk";
import zip from "./StreamToZip";
export default class HTMLExporter extends Exporter {
protected avatars: Map<string, boolean>;
protected permalinkCreator: RoomPermalinkCreator;
protected matrixClient: MatrixClient;
protected totalSize: number;
protected mediaOmitText: string;
constructor(room: Room, exportType: exportTypes, exportOptions: exportOptions) {
super(room, exportType, exportOptions);
this.avatars = new Map<string, boolean>();
this.matrixClient = MatrixClientPeg.get();
this.permalinkCreator = new RoomPermalinkCreator(this.room);
this.totalSize = 0;
this.mediaOmitText = !this.exportOptions.attachmentsIncluded
@ -78,7 +74,7 @@ export default class HTMLExporter extends Exporter {
const exportDate = formatFullDateNoDayNoTime(new Date());
const creator = this.room.currentState.getStateEvents(EventType.RoomCreate, "")?.getSender();
const creatorName = this.room?.getMember(creator)?.rawDisplayName || creator;
const exporter = this.matrixClient.getUserId();
const exporter = this.client.getUserId();
const exporterName = this.room?.getMember(exporter)?.rawDisplayName;
const topic = this.room.currentState.getStateEvents(EventType.RoomTopic, "")?.getContent()?.topic || "";
const createdText = _t("%(creatorName)s created this room.", {
@ -235,7 +231,7 @@ export default class HTMLExporter extends Exporter {
if (hasAvatar) await this.saveAvatarIfNeeded(mxEv);
const eventTile = <div className="mx_Export_EventWrapper" id={mxEv.getId()}>
<MatrixClientContext.Provider value = {this.matrixClient}>
<MatrixClientContext.Provider value = {this.client}>
<EventTile
mxEvent={mxEv}
continuation={continuation}

124
src/utils/exportUtils/JSONExport.ts Normal file
View file

@ -0,0 +1,124 @@
import streamSaver from "streamsaver";
import Exporter from "./Exporter";
import { Room } from "matrix-js-sdk/src/models/room";
import { MatrixEvent } from "matrix-js-sdk/src/models/event";
import { formatFullDateNoDay, formatFullDateNoDayNoTime } from "../../DateUtils";
import * as ponyfill from "web-streams-polyfill/ponyfill"
import { haveTileForEvent } from "../../components/views/rooms/EventTile";
import { exportTypes } from "./exportUtils";
import { exportOptions } from "./exportUtils";
import zip from "./StreamToZip";
import { EventType } from "../../../../matrix-js-sdk/src/@types/event";
export default class JSONExporter extends Exporter {
protected totalSize: number;
constructor(room: Room, exportType: exportTypes, exportOptions: exportOptions) {
super(room, exportType, exportOptions);
this.totalSize = 0;
window.addEventListener("beforeunload", this.onBeforeUnload)
}
protected wrapJSON(json: string): string {
const exportDate = formatFullDateNoDayNoTime(new Date());
const creator = this.room.currentState.getStateEvents(EventType.RoomCreate, "")?.getSender();
const creatorName = this.room?.getMember(creator)?.rawDisplayName || creator;
const topic = this.room.currentState.getStateEvents(EventType.RoomTopic, "")?.getContent()?.topic || "";
const exporter = this.client.getUserId();
const exporterName = this.room?.getMember(exporter)?.rawDisplayName || exporter;
return `{
"room_name": "${this.room.name}",
"room_creator": "${creatorName}",
"topic": "${topic}",
"export_date": "${exportDate}",
"exported_by": "${exporterName}",
"messages": [
${json}
]
}`
}
protected indentEachLine(string: string) {
const indent = ' ';
const regex = /^(?!\s*$)/gm;
return string.replace(regex, indent.repeat(1));
}
protected onBeforeUnload = (e: BeforeUnloadEvent) => {
e.preventDefault();
return e.returnValue = "Are you sure you want to exit during this export?";
}
protected async getJSONString(mxEv: MatrixEvent) {
if (this.exportOptions.attachmentsIncluded && this.isAttachment(mxEv)) {
try {
const blob = await this.getMediaBlob(mxEv);
this.totalSize += blob.size;
const filePath = this.getFilePath(mxEv);
this.addFile(filePath, blob);
if (this.totalSize > this.exportOptions.maxSize - 1024 * 1024) {
this.exportOptions.attachmentsIncluded = false;
}
} catch (err) {
console.log("Error fetching file: " + err);
}
}
const jsonEvent: any = mxEv.toJSON();
const clearEvent = mxEv.isEncrypted() ? jsonEvent.decrypted : jsonEvent;
const jsonString = JSON.stringify(clearEvent, null, 2);
return jsonString.length > 2 ? jsonString + ",\n" : "";
}
protected async createOutput(events: MatrixEvent[]) {
let content = "";
for (const event of events) {
if (!haveTileForEvent(event)) continue;
content += await this.getJSONString(event);
}
return this.wrapJSON(this.indentEachLine(content.slice(0, -2)));
}
public async export() {
console.info("Starting export process...");
console.info("Fetching events...");
const fetchStart = performance.now();
const res = await this.getRequiredEvents();
const fetchEnd = performance.now();
console.log(`Fetched ${res.length} events in ${(fetchEnd - fetchStart)/1000} s`);
console.info("Creating Output...");
const text = await this.createOutput(res);
console.info("Writing to the file system...");
streamSaver.WritableStream = ponyfill.WritableStream
const fileName = `matrix-export-${formatFullDateNoDay(new Date())}.json`;
const files = this.files;
if (files.length) {
this.addFile("result.json", new Blob([text]));
const fileStream = streamSaver.createWriteStream(fileName.slice(0, -5) + ".zip");
const readableZipStream = zip({
start(ctrl) {
for (const file of files) ctrl.enqueue(file);
ctrl.close();
},
});
const writer = fileStream.getWriter()
const reader = readableZipStream.getReader()
await this.pumpToFileStream(reader, writer);
} else {
const fileStream = streamSaver.createWriteStream(fileName);
const writer = fileStream.getWriter()
const data = new TextEncoder().encode(text);
await writer.write(data);
await writer.close();
}
const exportEnd = performance.now();
console.info(`Export Successful! Exported ${res.length} events in ${(exportEnd - fetchStart)/1000} seconds`);
window.removeEventListener("beforeunload", this.onBeforeUnload);
}
}

1
src/utils/exportUtils/PlainTextExport.ts
View file

@ -5,7 +5,6 @@ import { MatrixEvent } from "matrix-js-sdk/src/models/event";
import { formatFullDateNoDay } from "../../DateUtils";
import { _t } from "../../languageHandler";
import * as ponyfill from "web-streams-polyfill/ponyfill"
import "web-streams-polyfill/ponyfill" // to support blob.stream()
import { haveTileForEvent } from "../../components/views/rooms/EventTile";
import { exportTypes } from "./exportUtils";
import { exportOptions } from "./exportUtils";

2
src/utils/exportUtils/exportUtils.ts
View file

@ -1,5 +1,6 @@
import { Room } from "matrix-js-sdk/src/models/room";
import HTMLExporter from "./HtmlExport";
import JSONExporter from "./JSONExport";
import PlainTextExporter from "./PlainTextExport";
export enum exportFormats {
@ -33,6 +34,7 @@ const exportConversationalHistory = async (
await new HTMLExporter(room, exportType, exportOptions).export();
break;
case exportFormats.JSON:
await new JSONExporter(room, exportType, exportOptions).export();
break;
case exportFormats.LOGS:
await new PlainTextExporter(room, exportType, exportOptions).export();