Move away from streamsaver(for now)

This commit is contained in:
Jaiwanth 2021-07-26 22:15:05 +05:30
parent a4da8f9e9e
commit 41bc2b6481
4 changed files with 51 additions and 344 deletions

View file

@ -77,6 +77,7 @@
"highlight.js": "^10.5.0",
"html-entities": "^1.4.0",
"is-ip": "^3.1.0",
"jszip": "^3.7.0",
"katex": "^0.12.0",
"linkifyjs": "^2.1.9",
"lodash": "^4.17.20",
@ -99,10 +100,8 @@
"resize-observer-polyfill": "^1.5.1",
"rfc4648": "^1.4.0",
"sanitize-html": "^2.3.2",
"streamsaver": "^2.0.5",
"tar-js": "^0.3.0",
"url": "^0.11.0",
"web-streams-polyfill": "^3.0.3",
"what-input": "^5.2.10",
"zxcvbn": "^4.4.2"
},
@ -131,6 +130,7 @@
"@types/counterpart": "^0.18.1",
"@types/css-font-loading-module": "^0.0.6",
"@types/diff-match-patch": "^1.0.32",
"@types/file-saver": "^2.0.3",
"@types/flux": "^3.1.9",
"@types/jest": "^26.0.20",
"@types/linkifyjs": "^2.1.3",

View file

@ -1,4 +1,3 @@
import streamSaver from "streamsaver";
import { MatrixEvent } from "matrix-js-sdk/src/models/event";
import { Room } from "matrix-js-sdk/src/models/room";
import { MatrixClientPeg } from "../../MatrixClientPeg";
@ -7,21 +6,18 @@ import { decryptFile } from "../DecryptFile";
import { mediaFromContent } from "../../customisations/Media";
import { formatFullDateNoDay } from "../../DateUtils";
import { Direction, MatrixClient } from "matrix-js-sdk";
import streamToZIP from "./ZipStream";
import * as ponyfill from "web-streams-polyfill/ponyfill";
import "web-streams-polyfill/ponyfill"; // to support streams API for older browsers
import { MutableRefObject } from "react";
import JSZip from "jszip";
import { saveAs } from "file-saver";
type FileStream = {
type BlobFile = {
name: string;
stream(): ReadableStream;
blob: Blob;
};
export default abstract class Exporter {
protected files: FileStream[];
protected files: BlobFile[];
protected client: MatrixClient;
protected writer: WritableStreamDefaultWriter<any>;
protected fileStream: WritableStream<any>;
protected cancelled: boolean;
protected constructor(
@ -34,7 +30,6 @@ export default abstract class Exporter {
this.files = [];
this.client = MatrixClientPeg.get();
window.addEventListener("beforeunload", this.onBeforeUnload);
window.addEventListener("onunload", this.abortWriter);
}
protected onBeforeUnload(e: BeforeUnloadEvent): string {
@ -50,7 +45,7 @@ export default abstract class Exporter {
protected addFile(filePath: string, blob: Blob): void {
const file = {
name: filePath,
stream: () => blob.stream(),
blob,
};
this.files.push(file);
}
@ -58,67 +53,31 @@ export default abstract class Exporter {
protected async downloadZIP(): Promise<any> {
const filename = `matrix-export-${formatFullDateNoDay(new Date())}.zip`;
// Support for older browsers
streamSaver.WritableStream = ponyfill.WritableStream;
const zip = new JSZip();
// Create a writable stream to the directory
this.fileStream = streamSaver.createWriteStream(filename);
if (!this.cancelled) this.updateProgress("Generating a ZIP");
else return this.cleanUp();
this.writer = this.fileStream.getWriter();
const files = this.files;
for (const file of this.files) zip.file(file.name, file.blob);
const readableZipStream = streamToZIP({
start(ctrl) {
for (const file of files) ctrl.enqueue(file);
ctrl.close();
},
});
const content = await zip.generateAsync({ type: "blob" });
if (this.cancelled) return this.cleanUp();
this.updateProgress("Writing to the file system");
const reader = readableZipStream.getReader();
await this.pumpToFileStream(reader);
await saveAs(content, filename);
}
protected cleanUp(): string {
console.log("Cleaning up...");
window.removeEventListener("beforeunload", this.onBeforeUnload);
window.removeEventListener("onunload", this.abortWriter);
return "";
}
public async cancelExport(): Promise<void> {
console.log("Cancelling export...");
this.cancelled = true;
await this.abortWriter();
}
protected async downloadPlainText(fileName: string, text: string): Promise<any> {
this.fileStream = streamSaver.createWriteStream(fileName);
this.writer = this.fileStream.getWriter();
const data = new TextEncoder().encode(text);
if (this.cancelled) return this.cleanUp();
await this.writer.write(data);
await this.writer.close();
}
protected async abortWriter(): Promise<void> {
await this.fileStream?.abort();
await this.writer?.abort();
}
protected async pumpToFileStream(reader: ReadableStreamDefaultReader): Promise<void> {
const res = await reader.read();
if (res.done) await this.writer.close();
else {
await this.writer.write(res.value);
await this.pumpToFileStream(reader);
}
await saveAs(new Blob[text], fileName);
}
protected setEventMetadata(event: MatrixEvent): MatrixEvent {

View file

@ -1,279 +0,0 @@
// Based on https://github.com/jimmywarting/StreamSaver.js/blob/master/examples/zip-stream.js
/* global ReadableStream */
type TypedArray =
| Int8Array
| Uint8Array
| Int16Array
| Uint16Array
| Int32Array
| Uint32Array
| Uint8ClampedArray
| Float32Array
| Float64Array;
/**
* 32-bit cyclic redundancy check, or CRC-32 - checksum
*/
class Crc32 {
crc: number;
table: any;
constructor() {
this.crc = -1;
this.table = (() => {
let i;
let j;
let t;
const table = [];
for (i = 0; i < 256; i++) {
t = i;
for (j = 0; j < 8; j++) {
t = (t & 1)
? (t >>> 1) ^ 0xEDB88320
: t >>> 1;
}
table[i] = t;
}
return table;
})();
}
append(data: TypedArray) {
let crc = this.crc | 0;
const table = this.table;
for (let offset = 0, len = data.length | 0; offset < len; offset++) {
crc = (crc >>> 8) ^ table[(crc ^ data[offset]) & 0xFF];
}
this.crc = crc;
}
get() {
return ~this.crc;
}
}
type DataHelper = {
array: Uint8Array;
view: DataView;
};
const getDataHelper = (byteLength: number): DataHelper => {
const uint8 = new Uint8Array(byteLength);
return {
array: uint8,
view: new DataView(uint8.buffer),
};
};
type FileLike = File & {
directory: string;
comment: string;
stream(): ReadableStream;
};
type ZipObj = {
crc?: Crc32;
uncompressedLength: number;
compressedLength: number;
ctrl: ReadableStreamDefaultController;
writeFooter: Function;
writeHeader: Function;
reader?: ReadableStreamDefaultReader;
offset: number;
header?: DataHelper;
fileLike: FileLike;
level: number;
directory: boolean;
};
const pump = (zipObj: ZipObj) => zipObj.reader ? zipObj.reader.read().then(chunk => {
if (zipObj.crc) {
if (chunk.done) return zipObj.writeFooter();
const outputData = chunk.value;
zipObj.crc.append(outputData);
zipObj.uncompressedLength += outputData.length;
zipObj.compressedLength += outputData.length;
zipObj.ctrl.enqueue(outputData);
} else {
throw new Error('Missing zipObj.crc');
}
}) : undefined;
export default function streamToZIP(underlyingSource: UnderlyingSource) {
const files = Object.create(null);
const filenames: string[] = [];
const encoder = new TextEncoder();
let offset = 0;
let activeZipIndex = 0;
let ctrl: ReadableStreamDefaultController;
let activeZipObject: ZipObj;
let closed: boolean;
function next() {
activeZipIndex++;
activeZipObject = files[filenames[activeZipIndex]];
if (activeZipObject) processNextChunk();
else if (closed) closeZip();
}
const zipWriter: ReadableStreamDefaultController = {
desiredSize: null,
error(err) {
console.error(err);
},
enqueue(fileLike: FileLike) {
if (closed) {
throw new TypeError(
"Cannot enqueue a chunk into a readable stream that is closed or has been requested to be closed",
);
}
let name = fileLike.name.trim();
const date = new Date(typeof fileLike.lastModified === 'undefined' ? Date.now() : fileLike.lastModified);
if (fileLike.directory && !name.endsWith('/')) name += '/';
// if file already exists, do not enqueue
if (files[name]) return;
const nameBuf = encoder.encode(name);
filenames.push(name);
const zipObject: ZipObj = files[name] = {
level: 0,
ctrl,
directory: !!fileLike.directory,
nameBuf,
comment: encoder.encode(fileLike.comment || ''),
compressedLength: 0,
uncompressedLength: 0,
offset,
writeHeader() {
const header = getDataHelper(26);
const data = getDataHelper(30 + nameBuf.length);
zipObject.offset = offset;
zipObject.header = header;
if (zipObject.level !== 0 && !zipObject.directory) {
header.view.setUint16(4, 0x0800);
}
header.view.setUint32(0, 0x14000808);
header.view.setUint16(
6,
(((date.getHours() << 6) | date.getMinutes()) << 5) | (date.getSeconds() / 2),
true,
);
header.view.setUint16(
8,
((((date.getFullYear() - 1980) << 4) | (date.getMonth() + 1)) << 5) |
date.getDate(),
true,
);
header.view.setUint16(22, nameBuf.length, true);
data.view.setUint32(0, 0x504b0304);
data.array.set(header.array, 4);
data.array.set(nameBuf, 30);
offset += data.array.length;
ctrl.enqueue(data.array);
},
writeFooter() {
const footer = getDataHelper(16);
footer.view.setUint32(0, 0x504b0708);
if (zipObject.crc && zipObject.header) {
zipObject.header.view.setUint32(10, zipObject.crc.get(), true);
zipObject.header.view.setUint32(14, zipObject.compressedLength, true);
zipObject.header.view.setUint32(18, zipObject.uncompressedLength, true);
footer.view.setUint32(4, zipObject.crc.get(), true);
footer.view.setUint32(8, zipObject.compressedLength, true);
footer.view.setUint32(12, zipObject.uncompressedLength, true);
}
ctrl.enqueue(footer.array);
offset += zipObject.compressedLength + 16;
next();
},
fileLike,
};
if (!activeZipObject) {
activeZipObject = zipObject;
processNextChunk();
}
},
close() {
if (closed) {
throw new TypeError(
"Cannot close a readable stream that has already been requested to be closed",
);
}
if (!activeZipObject) closeZip();
closed = true;
},
};
function closeZip() {
let length = 0;
let index = 0;
let indexFilename: number;
let file: any;
for (indexFilename = 0; indexFilename < filenames.length; indexFilename++) {
file = files[filenames[indexFilename]];
length += 46 + file.nameBuf.length + file.comment.length;
}
const data = getDataHelper(length + 22);
for (indexFilename = 0; indexFilename < filenames.length; indexFilename++) {
file = files[filenames[indexFilename]];
data.view.setUint32(index, 0x504b0102);
data.view.setUint16(index + 4, 0x1400);
data.array.set(file.header.array, index + 6);
data.view.setUint16(index + 32, file.comment.length, true);
if (file.directory) {
data.view.setUint8(index + 38, 0x10);
}
data.view.setUint32(index + 42, file.offset, true);
data.array.set(file.nameBuf, index + 46);
data.array.set(file.comment, index + 46 + file.nameBuf.length);
index += 46 + file.nameBuf.length + file.comment.length;
}
data.view.setUint32(index, 0x504b0506);
data.view.setUint16(index + 8, filenames.length, true);
data.view.setUint16(index + 10, filenames.length, true);
data.view.setUint32(index + 12, length, true);
data.view.setUint32(index + 16, offset, true);
ctrl.enqueue(data.array);
ctrl.close();
}
function processNextChunk() {
if (!activeZipObject) return;
if (activeZipObject.reader) return pump(activeZipObject);
if (activeZipObject.fileLike.stream) {
activeZipObject.crc = new Crc32();
activeZipObject.reader = activeZipObject.fileLike.stream().getReader();
activeZipObject.writeHeader();
} else next();
}
return new ReadableStream({
start: c => {
ctrl = c;
underlyingSource.start && Promise.resolve(underlyingSource.start(zipWriter));
},
pull() {
return processNextChunk() || (
underlyingSource.pull &&
Promise.resolve(underlyingSource.pull(zipWriter))
);
},
});
}

View file

@ -1468,6 +1468,11 @@
resolved "https://registry.yarnpkg.com/@types/fbemitter/-/fbemitter-2.0.32.tgz#8ed204da0f54e9c8eaec31b1eec91e25132d082c"
integrity sha1-jtIE2g9U6cjq7DGx7skeJRMtCCw=
"@types/file-saver@^2.0.3":
version "2.0.3"
resolved "https://registry.yarnpkg.com/@types/file-saver/-/file-saver-2.0.3.tgz#b734c4f5a04d20615eaed3dc106e2ab321082009"
integrity sha512-MBIou8pd/41jkff7s97B47bc9+p0BszqqDJsO51yDm49uUxeKzrfuNl5fSLC6BpLEWKA8zlwyqALVmXrFwoBHQ==
"@types/flux@^3.1.9":
version "3.1.10"
resolved "https://registry.yarnpkg.com/@types/flux/-/flux-3.1.10.tgz#7c6306e86ecb434d00f38cb82f092640c7bd4098"
@ -4141,6 +4146,11 @@ ignore@^5.1.4, ignore@^5.1.8:
resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57"
integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==
immediate@~3.0.5:
version "3.0.6"
resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
integrity sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=
immutable@^3.7.4:
version "3.8.2"
resolved "https://registry.yarnpkg.com/immutable/-/immutable-3.8.2.tgz#c2439951455bb39913daf281376f1530e104adf3"
@ -5235,6 +5245,16 @@ jsprim@^1.2.2:
array-includes "^3.1.2"
object.assign "^4.1.2"
jszip@^3.7.0:
version "3.7.0"
resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.7.0.tgz#9b8b995a4e7c9024653ce743e902076a82fdf4e6"
integrity sha512-Y2OlFIzrDOPWUnpU0LORIcDn2xN7rC9yKffFM/7pGhQuhO+SUhfm2trkJ/S5amjFvem0Y+1EALz/MEPkvHXVNw==
dependencies:
lie "~3.3.0"
pako "~1.0.2"
readable-stream "~2.3.6"
set-immediate-shim "~1.0.1"
katex@^0.12.0:
version "0.12.0"
resolved "https://registry.yarnpkg.com/katex/-/katex-0.12.0.tgz#2fb1c665dbd2b043edcf8a1f5c555f46beaa0cb9"
@ -5302,6 +5322,13 @@ levn@~0.3.0:
prelude-ls "~1.1.2"
type-check "~0.3.2"
lie@~3.3.0:
version "3.3.0"
resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a"
integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==
dependencies:
immediate "~3.0.5"
lines-and-columns@^1.1.6:
version "1.1.6"
resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00"
@ -6015,6 +6042,11 @@ pako@^2.0.3:
resolved "https://registry.yarnpkg.com/pako/-/pako-2.0.3.tgz#cdf475e31b678565251406de9e759196a0ea7a43"
integrity sha512-WjR1hOeg+kki3ZIOjaf4b5WVcay1jaliKSYiEaB1XzwhMQZJxRdQRv0V31EKBYlxb4T7SK3hjfc/jxyU64BoSw==
pako@~1.0.2:
version "1.0.11"
resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"
integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==
parent-module@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2"
@ -6552,7 +6584,7 @@ read-pkg@^5.2.0:
parse-json "^5.0.0"
type-fest "^0.6.0"
readable-stream@^2.0.2:
readable-stream@^2.0.2, readable-stream@~2.3.6:
version "2.3.7"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==
@ -6953,6 +6985,11 @@ set-blocking@^2.0.0:
resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc=
set-immediate-shim@~1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz#4b2b1b27eb808a9f8dcc481a58e5e56f599f3f61"
integrity sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=
set-value@^2.0.0, set-value@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/set-value/-/set-value-2.0.1.tgz#a18d40530e6f07de4228c7defe4227af8cad005b"
@ -7206,11 +7243,6 @@ static-extend@^0.1.1:
define-property "^0.2.5"
object-copy "^0.1.0"
streamsaver@^2.0.5:
version "2.0.5"
resolved "https://registry.yarnpkg.com/streamsaver/-/streamsaver-2.0.5.tgz#3212f0e908fcece5b3a65591094475cf87850d00"
integrity sha512-KIWtBvi8A6FiFZGNSyuIZRZM6C8AvnWTiCx/TYa7so420vC5sQwcBKkdqInuGWoWMfeWy/P+/cRqMtWVf4RW9w==
string-length@^4.0.1:
version "4.0.2"
resolved "https://registry.yarnpkg.com/string-length/-/string-length-4.0.2.tgz#a8a8dc7bd5c1a82b9b3c8b87e125f66871b6e57a"
@ -7916,11 +7948,6 @@ walker@^1.0.7, walker@~1.0.5:
dependencies:
makeerror "1.0.x"
web-streams-polyfill@^3.0.3:
version "3.0.3"
resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.0.3.tgz#f49e487eedeca47a207c1aee41ee5578f884b42f"
integrity sha512-d2H/t0eqRNM4w2WvmTdoeIvzAUSpK7JmATB8Nr2lb7nQ9BTIJVjbQ/TRFVEh2gUH1HwclPdoPtfMoFfetXaZnA==
webcrypto-core@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/webcrypto-core/-/webcrypto-core-1.2.0.tgz#44fda3f9315ed6effe9a1e47466e0935327733b5"