Allow image pasting in rich text mode in RTE (#11049)

* add comments to rough first solution

* allow eventRelation prop to pass to both composers

* use eventRelation in image paste

* add image pasting to rich text mode of rich text editor

* extract error handling to function

* type the error handler

* add tests

* make behaviour mimic SendMessage

* add sad path tests

* refactor to use catch throughout

* update comments

* tidy up tests

* add special case and change function signature

* add comment

* bump rte to 2.2.2
This commit is contained in:
alunturner 2023-06-09 10:11:38 +01:00 committed by GitHub
parent 72e6c10f0d
commit 53415bfdfe
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 414 additions and 14 deletions

View file

@ -61,7 +61,7 @@
"dependencies": {
"@babel/runtime": "^7.12.5",
"@matrix-org/analytics-events": "^0.5.0",
"@matrix-org/matrix-wysiwyg": "^2.0.0",
"@matrix-org/matrix-wysiwyg": "^2.2.2",
"@matrix-org/react-sdk-module-api": "^0.0.5",
"@sentry/browser": "^7.0.0",
"@sentry/tracing": "^7.0.0",

View file

@ -57,11 +57,10 @@ export default function SendWysiwygComposer({
isRichTextEnabled,
e2eStatus,
menuPosition,
eventRelation,
...props
}: SendWysiwygComposerProps): JSX.Element {
const Composer = isRichTextEnabled ? WysiwygComposer : PlainTextComposer;
const defaultContextValue = useRef(getDefaultContextValue({ eventRelation }));
const defaultContextValue = useRef(getDefaultContextValue({ eventRelation: props.eventRelation }));
return (
<ComposerContext.Provider value={defaultContextValue.current}>

View file

@ -15,6 +15,7 @@ limitations under the License.
*/
import classNames from "classnames";
import { IEventRelation } from "matrix-js-sdk/src/matrix";
import React, { MutableRefObject, ReactNode } from "react";
import { useComposerFunctions } from "../hooks/useComposerFunctions";
@ -36,6 +37,7 @@ interface PlainTextComposerProps {
leftComponent?: ReactNode;
rightComponent?: ReactNode;
children?: (ref: MutableRefObject<HTMLDivElement | null>, composerFunctions: ComposerFunctions) => ReactNode;
eventRelation?: IEventRelation;
}
export function PlainTextComposer({

View file

@ -15,6 +15,7 @@ limitations under the License.
*/
import React, { memo, MutableRefObject, ReactNode, useEffect, useRef } from "react";
import { IEventRelation } from "matrix-js-sdk/src/matrix";
import { useWysiwyg, FormattingFunctions } from "@matrix-org/matrix-wysiwyg";
import classNames from "classnames";
@ -40,6 +41,7 @@ interface WysiwygComposerProps {
leftComponent?: ReactNode;
rightComponent?: ReactNode;
children?: (ref: MutableRefObject<HTMLDivElement | null>, wysiwyg: FormattingFunctions) => ReactNode;
eventRelation?: IEventRelation;
}
export const WysiwygComposer = memo(function WysiwygComposer({
@ -52,11 +54,12 @@ export const WysiwygComposer = memo(function WysiwygComposer({
leftComponent,
rightComponent,
children,
eventRelation,
}: WysiwygComposerProps) {
const { room } = useRoomContext();
const autocompleteRef = useRef<Autocomplete | null>(null);
const inputEventProcessor = useInputEventProcessor(onSend, autocompleteRef, initialContent);
const inputEventProcessor = useInputEventProcessor(onSend, autocompleteRef, initialContent, eventRelation);
const { ref, isWysiwygReady, content, actionStates, wysiwyg, suggestion } = useWysiwyg({
initialContent,
inputEventProcessor,

View file

@ -16,7 +16,7 @@ limitations under the License.
import { Wysiwyg, WysiwygEvent } from "@matrix-org/matrix-wysiwyg";
import { useCallback } from "react";
import { MatrixClient } from "matrix-js-sdk/src/matrix";
import { IEventRelation, MatrixClient } from "matrix-js-sdk/src/matrix";
import { useSettingValue } from "../../../../../hooks/useSettings";
import { getKeyBindingsManager } from "../../../../../KeyBindingsManager";
@ -34,11 +34,15 @@ import { getEventsFromEditorStateTransfer, getEventsFromRoom } from "../utils/ev
import { endEditing } from "../utils/editing";
import Autocomplete from "../../Autocomplete";
import { handleEventWithAutocomplete } from "./utils";
import ContentMessages from "../../../../../ContentMessages";
import { getBlobSafeMimeType } from "../../../../../utils/blobs";
import { isNotNull } from "../../../../../Typeguards";
export function useInputEventProcessor(
onSend: () => void,
autocompleteRef: React.RefObject<Autocomplete>,
initialContent?: string,
eventRelation?: IEventRelation,
): (event: WysiwygEvent, composer: Wysiwyg, editor: HTMLElement) => WysiwygEvent | null {
const roomContext = useRoomContext();
const composerContext = useComposerContext();
@ -47,10 +51,6 @@ export function useInputEventProcessor(
return useCallback(
(event: WysiwygEvent, composer: Wysiwyg, editor: HTMLElement) => {
if (event instanceof ClipboardEvent) {
return event;
}
const send = (): void => {
event.stopPropagation?.();
event.preventDefault?.();
@ -61,6 +61,21 @@ export function useInputEventProcessor(
onSend();
};
// this is required to handle edge case image pasting in Safari, see
// https://github.com/vector-im/element-web/issues/25327 and it is caught by the
// `beforeinput` listener attached to the composer
const isInputEventForClipboard =
event instanceof InputEvent && event.inputType === "insertFromPaste" && isNotNull(event.dataTransfer);
const isClipboardEvent = event instanceof ClipboardEvent;
const shouldHandleAsClipboardEvent = isClipboardEvent || isInputEventForClipboard;
if (shouldHandleAsClipboardEvent) {
const data = isClipboardEvent ? event.clipboardData : event.dataTransfer;
const handled = handleClipboardEvent(event, data, roomContext, mxClient, eventRelation);
return handled ? null : event;
}
const isKeyboardEvent = event instanceof KeyboardEvent;
if (isKeyboardEvent) {
return handleKeyboardEvent(
@ -78,7 +93,16 @@ export function useInputEventProcessor(
return handleInputEvent(event, send, isCtrlEnterToSend);
}
},
[isCtrlEnterToSend, onSend, initialContent, roomContext, composerContext, mxClient, autocompleteRef],
[
isCtrlEnterToSend,
onSend,
initialContent,
roomContext,
composerContext,
mxClient,
autocompleteRef,
eventRelation,
],
);
}
@ -220,3 +244,88 @@ function handleInputEvent(event: InputEvent, send: Send, isCtrlEnterToSend: bool
return event;
}
/**
* Takes an event and handles image pasting. Returns a boolean to indicate if it has handled
* the event or not. Must accept either clipboard or input events in order to prevent issue:
* https://github.com/vector-im/element-web/issues/25327
*
* @param event - event to process
* @param roomContext - room in which the event occurs
* @param mxClient - current matrix client
* @param eventRelation - used to send the event to the correct place eg timeline vs thread
* @returns - boolean to show if the event was handled or not
*/
export function handleClipboardEvent(
event: ClipboardEvent | InputEvent,
data: DataTransfer | null,
roomContext: IRoomState,
mxClient: MatrixClient,
eventRelation?: IEventRelation,
): boolean {
// Logic in this function follows that of `SendMessageComposer.onPaste`
const { room, timelineRenderingType, replyToEvent } = roomContext;
function handleError(error: unknown): void {
if (error instanceof Error) {
console.log(error.message);
} else if (typeof error === "string") {
console.log(error);
}
}
if (event.type !== "paste" || data === null || room === undefined) {
return false;
}
// Prioritize text on the clipboard over files if RTF is present as Office on macOS puts a bitmap
// in the clipboard as well as the content being copied. Modern versions of Office seem to not do this anymore.
// We check text/rtf instead of text/plain as when copy+pasting a file from Finder or Gnome Image Viewer
// it puts the filename in as text/plain which we want to ignore.
if (data.files.length && !data.types.includes("text/rtf")) {
ContentMessages.sharedInstance()
.sendContentListToRoom(Array.from(data.files), room.roomId, eventRelation, mxClient, timelineRenderingType)
.catch(handleError);
return true;
}
// Safari `Insert from iPhone or iPad`
// data.getData("text/html") returns a string like: <img src="blob:https://...">
if (data.types.includes("text/html")) {
const imgElementStr = data.getData("text/html");
const parser = new DOMParser();
const imgDoc = parser.parseFromString(imgElementStr, "text/html");
if (
imgDoc.getElementsByTagName("img").length !== 1 ||
!imgDoc.querySelector("img")?.src.startsWith("blob:") ||
imgDoc.childNodes.length !== 1
) {
handleError("Failed to handle pasted content as Safari inserted content");
return false;
}
const imgSrc = imgDoc.querySelector("img")!.src;
fetch(imgSrc)
.then((response) => {
response
.blob()
.then((imgBlob) => {
const type = imgBlob.type;
const safetype = getBlobSafeMimeType(type);
const ext = type.split("/")[1];
const parts = response.url.split("/");
const filename = parts[parts.length - 1];
const file = new File([imgBlob], filename + "." + ext, { type: safetype });
ContentMessages.sharedInstance()
.sendContentToRoom(file, room.roomId, eventRelation, mxClient, replyToEvent)
.catch(handleError);
})
.catch(handleError);
})
.catch(handleError);
return true;
}
return false;
}

View file

@ -0,0 +1,287 @@
/*
Copyright 2023 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { IEventRelation, MatrixEvent } from "matrix-js-sdk/src/matrix";
import { waitFor } from "@testing-library/react";
import { handleClipboardEvent } from "../../../../../../src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor";
import { TimelineRenderingType } from "../../../../../../src/contexts/RoomContext";
import { mkStubRoom, stubClient } from "../../../../../test-utils";
import ContentMessages from "../../../../../../src/ContentMessages";
import { IRoomState } from "../../../../../../src/components/structures/RoomView";
const mockClient = stubClient();
const mockRoom = mkStubRoom("mock room", "mock room", mockClient);
const mockRoomState = {
room: mockRoom,
timelineRenderingType: TimelineRenderingType.Room,
replyToEvent: {} as unknown as MatrixEvent,
} as unknown as IRoomState;
const sendContentListToRoomSpy = jest.spyOn(ContentMessages.sharedInstance(), "sendContentListToRoom");
const sendContentToRoomSpy = jest.spyOn(ContentMessages.sharedInstance(), "sendContentToRoom");
const fetchSpy = jest.spyOn(window, "fetch");
const logSpy = jest.spyOn(console, "log").mockImplementation(() => {});
describe("handleClipboardEvent", () => {
beforeEach(() => {
jest.clearAllMocks();
});
afterAll(() => {
jest.restoreAllMocks();
});
function createMockClipboardEvent(props: any): ClipboardEvent {
return { clipboardData: { files: [], types: [] }, ...props } as ClipboardEvent;
}
it("returns false if it is not a paste event", () => {
const originalEvent = createMockClipboardEvent({ type: "copy" });
const output = handleClipboardEvent(originalEvent, originalEvent.clipboardData, mockRoomState, mockClient);
expect(output).toBe(false);
});
it("returns false if clipboard data is null", () => {
const originalEvent = createMockClipboardEvent({ type: "paste", clipboardData: null });
const output = handleClipboardEvent(originalEvent, originalEvent.clipboardData, mockRoomState, mockClient);
expect(output).toBe(false);
});
it("returns false if room is undefined", () => {
const originalEvent = createMockClipboardEvent({ type: "paste" });
const { room, ...roomStateWithoutRoom } = mockRoomState;
const output = handleClipboardEvent(
originalEvent,
originalEvent.clipboardData,
roomStateWithoutRoom,
mockClient,
);
expect(output).toBe(false);
});
it("returns false if room clipboardData files and types are empty", () => {
const originalEvent = createMockClipboardEvent({
type: "paste",
clipboardData: { files: [], types: [] },
});
const output = handleClipboardEvent(originalEvent, originalEvent.clipboardData, mockRoomState, mockClient);
expect(output).toBe(false);
});
it("handles event and calls sendContentListToRoom when data files are present", () => {
const originalEvent = createMockClipboardEvent({
type: "paste",
clipboardData: { files: ["something here"], types: [] },
});
const output = handleClipboardEvent(originalEvent, originalEvent.clipboardData, mockRoomState, mockClient);
expect(sendContentListToRoomSpy).toHaveBeenCalledTimes(1);
expect(sendContentListToRoomSpy).toHaveBeenCalledWith(
originalEvent.clipboardData?.files,
mockRoom.roomId,
undefined, // this is the event relation, an optional arg
mockClient,
mockRoomState.timelineRenderingType,
);
expect(output).toBe(true);
});
it("calls sendContentListToRoom with eventRelation when present", () => {
const originalEvent = createMockClipboardEvent({
type: "paste",
clipboardData: { files: ["something here"], types: [] },
});
const mockEventRelation = {} as unknown as IEventRelation;
const output = handleClipboardEvent(
originalEvent,
originalEvent.clipboardData,
mockRoomState,
mockClient,
mockEventRelation,
);
expect(sendContentListToRoomSpy).toHaveBeenCalledTimes(1);
expect(sendContentListToRoomSpy).toHaveBeenCalledWith(
originalEvent.clipboardData?.files,
mockRoom.roomId,
mockEventRelation, // this is the event relation, an optional arg
mockClient,
mockRoomState.timelineRenderingType,
);
expect(output).toBe(true);
});
it("calls the error handler when sentContentListToRoom errors", async () => {
const mockErrorMessage = "something went wrong";
sendContentListToRoomSpy.mockRejectedValueOnce(new Error(mockErrorMessage));
const originalEvent = createMockClipboardEvent({
type: "paste",
clipboardData: { files: ["something here"], types: [] },
});
const mockEventRelation = {} as unknown as IEventRelation;
const output = handleClipboardEvent(
originalEvent,
originalEvent.clipboardData,
mockRoomState,
mockClient,
mockEventRelation,
);
expect(sendContentListToRoomSpy).toHaveBeenCalledTimes(1);
await waitFor(() => {
expect(logSpy).toHaveBeenCalledWith(mockErrorMessage);
});
expect(output).toBe(true);
});
it("calls the error handler when data types has text/html but data can not be parsed", () => {
const originalEvent = createMockClipboardEvent({
type: "paste",
clipboardData: {
files: [],
types: ["text/html"],
getData: jest.fn().mockReturnValue("<div>invalid html"),
},
});
const mockEventRelation = {} as unknown as IEventRelation;
const output = handleClipboardEvent(
originalEvent,
originalEvent.clipboardData,
mockRoomState,
mockClient,
mockEventRelation,
);
expect(logSpy).toHaveBeenCalledWith("Failed to handle pasted content as Safari inserted content");
expect(output).toBe(false);
});
it("calls fetch when data types has text/html and data can parsed", () => {
const originalEvent = createMockClipboardEvent({
type: "paste",
clipboardData: {
files: [],
types: ["text/html"],
getData: jest.fn().mockReturnValue(`<img src="blob:" />`),
},
});
const mockEventRelation = {} as unknown as IEventRelation;
handleClipboardEvent(originalEvent, originalEvent.clipboardData, mockRoomState, mockClient, mockEventRelation);
expect(fetchSpy).toHaveBeenCalledTimes(1);
expect(fetchSpy).toHaveBeenCalledWith("blob:");
});
it("calls error handler when fetch fails", async () => {
const mockErrorMessage = "fetch failed";
fetchSpy.mockRejectedValueOnce(mockErrorMessage);
const originalEvent = createMockClipboardEvent({
type: "paste",
clipboardData: {
files: [],
types: ["text/html"],
getData: jest.fn().mockReturnValue(`<img src="blob:" />`),
},
});
const mockEventRelation = {} as unknown as IEventRelation;
const output = handleClipboardEvent(
originalEvent,
originalEvent.clipboardData,
mockRoomState,
mockClient,
mockEventRelation,
);
await waitFor(() => {
expect(logSpy).toHaveBeenCalledWith(mockErrorMessage);
});
expect(output).toBe(true);
});
it("calls sendContentToRoom when parsing is successful", async () => {
fetchSpy.mockResolvedValueOnce({
url: "test/file",
blob: () => {
return Promise.resolve({ type: "image/jpeg" } as Blob);
},
} as Response);
const originalEvent = createMockClipboardEvent({
type: "paste",
clipboardData: {
files: [],
types: ["text/html"],
getData: jest.fn().mockReturnValue(`<img src="blob:" />`),
},
});
const mockEventRelation = {} as unknown as IEventRelation;
const output = handleClipboardEvent(
originalEvent,
originalEvent.clipboardData,
mockRoomState,
mockClient,
mockEventRelation,
);
await waitFor(() => {
expect(sendContentToRoomSpy).toHaveBeenCalledWith(
expect.any(File),
mockRoom.roomId,
mockEventRelation,
mockClient,
mockRoomState.replyToEvent,
);
});
expect(output).toBe(true);
});
it("calls error handler when parsing is not successful", async () => {
fetchSpy.mockResolvedValueOnce({
url: "test/file",
blob: () => {
return Promise.resolve({ type: "image/jpeg" } as Blob);
},
} as Response);
const mockErrorMessage = "sendContentToRoom failed";
sendContentToRoomSpy.mockRejectedValueOnce(mockErrorMessage);
const originalEvent = createMockClipboardEvent({
type: "paste",
clipboardData: {
files: [],
types: ["text/html"],
getData: jest.fn().mockReturnValue(`<img src="blob:" />`),
},
});
const mockEventRelation = {} as unknown as IEventRelation;
const output = handleClipboardEvent(
originalEvent,
originalEvent.clipboardData,
mockRoomState,
mockClient,
mockEventRelation,
);
await waitFor(() => {
expect(logSpy).toHaveBeenCalledWith(mockErrorMessage);
});
expect(output).toBe(true);
});
});

View file

@ -1598,10 +1598,10 @@
resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-js/-/matrix-sdk-crypto-js-0.1.0-alpha.10.tgz#b6a6395cffd3197ae2e0a88f4eeae8b315571fd2"
integrity sha512-8V2NKuzGOFzEZeZVgF2is7gmuopdRbMZ064tzPDE0vN34iX6s3O8A4oxIT7SA3qtymwm3t1yEvTnT+0gfbmh4g==
"@matrix-org/matrix-wysiwyg@^2.0.0":
version "2.2.1"
resolved "https://registry.yarnpkg.com/@matrix-org/matrix-wysiwyg/-/matrix-wysiwyg-2.2.1.tgz#076b409c0ffe655938d663863b1ee546a7101da6"
integrity sha512-QF4dJsyqBMxZx+GhSdSiRSDIuwE5dxd7vffQ5i6hf67bd0EbVvtf4PzWmNopGHA+ckjMJIc5X1EPT+6DG/wM6Q==
"@matrix-org/matrix-wysiwyg@^2.2.2":
version "2.2.2"
resolved "https://registry.yarnpkg.com/@matrix-org/matrix-wysiwyg/-/matrix-wysiwyg-2.2.2.tgz#911d0a9858a5a4b620f93777085daac8eff6a220"
integrity sha512-FprkgKiqEHoFUfaamKwTGBENqDxbORFgoPjiE1b9yPS3hgRswobVKRl4qrXgVgFj4qQ7gWeTqogiyrHXkm1myw==
"@matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz":
version "3.2.14"