From f04a0e2860a2c271160fd1afa7743ba79c78d474 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Mon, 17 Jul 2023 13:07:58 +0100 Subject: [PATCH] Populate info.duration for audio & video file uploads (#11225) * Improve m.file m.image m.audio m.video types * Populate `info.duration` for audio & video file uploads * Fix tests * Iterate types * Improve coverage * Fix test * Add small delay to stabilise cypress test * Fix test idempotency * Improve coverage * Slow down * iterate --- cypress/e2e/right-panel/file-panel.spec.ts | 6 +- src/ContentMessages.ts | 87 +++++-- src/components/views/messages/MFileBody.tsx | 2 +- src/components/views/messages/MImageBody.tsx | 22 +- .../views/messages/MImageReplyBody.tsx | 4 +- .../models/IMediaEventContent.ts | 223 ++++++++++++++++-- src/models/RoomUpload.ts | 4 +- src/utils/DecryptFile.ts | 6 +- src/utils/MediaEventHelper.ts | 6 +- src/utils/image-media.ts | 4 +- .../models/VoiceBroadcastRecording.ts | 4 +- test/ContentMessages-test.ts | 53 +++++ .../views/messages/MImageBody-test.tsx | 68 +++++- .../views/messages/MVideoBody-test.tsx | 71 +++++- .../__snapshots__/MImageBody-test.tsx.snap | 50 ++++ .../__snapshots__/MVideoBody-test.tsx.snap | 25 ++ .../models/VoiceBroadcastRecording-test.ts | 6 +- 17 files changed, 556 insertions(+), 85 deletions(-) diff --git a/cypress/e2e/right-panel/file-panel.spec.ts b/cypress/e2e/right-panel/file-panel.spec.ts index fa436772ba..892e3d94b3 100644 --- a/cypress/e2e/right-panel/file-panel.spec.ts +++ b/cypress/e2e/right-panel/file-panel.spec.ts @@ -183,7 +183,7 @@ describe("FilePanel", () => { }); }); - it("should render the audio pleyer and play the audio file on the panel", () => { + it("should render the audio player and play the audio file on the panel", () => { // Upload an image file uploadFile("cypress/fixtures/1sec.ogg"); @@ -202,10 +202,14 @@ describe("FilePanel", () => { cy.contains(".mx_AudioPlayer_byline", "(3.56 KB)").should("exist"); // actual size }); + // Assert that the duration counter is 00:01 before clicking the play button + cy.contains(".mx_AudioPlayer_mediaInfo time", "00:01").should("exist"); + // Assert that the counter is zero before clicking the play button cy.contains(".mx_AudioPlayer_seek [role='timer']", "00:00").should("exist"); // Click the play button + cy.wait(500); cy.findByRole("button", { name: "Play" }).click(); // Assert that the pause button is rendered diff --git a/src/ContentMessages.ts b/src/ContentMessages.ts index f7f6b148e2..8608f23bf0 100644 --- a/src/ContentMessages.ts +++ b/src/ContentMessages.ts @@ -33,7 +33,14 @@ import { import { THREAD_RELATION_TYPE } from "matrix-js-sdk/src/models/thread"; import { removeElement } from "matrix-js-sdk/src/utils"; -import { IEncryptedFile, IMediaEventContent, IMediaEventInfo } from "./customisations/models/IMediaEventContent"; +import { + AudioInfo, + EncryptedFile, + ImageInfo, + IMediaEventContent, + IMediaEventInfo, + VideoInfo, +} from "./customisations/models/IMediaEventContent"; import dis from "./dispatcher/dispatcher"; import { _t } from "./languageHandler"; import Modal from "./Modal"; @@ -146,11 +153,7 @@ const ALWAYS_INCLUDE_THUMBNAIL = ["image/avif", "image/webp"]; * @param {File} imageFile The image to read and thumbnail. * @return {Promise} A promise that resolves with the attachment info. */ -async function infoForImageFile( - matrixClient: MatrixClient, - roomId: string, - imageFile: File, -): Promise> { +async function infoForImageFile(matrixClient: MatrixClient, roomId: string, imageFile: File): Promise { let thumbnailType = "image/png"; if (imageFile.type === "image/jpeg") { thumbnailType = "image/jpeg"; @@ -184,16 +187,59 @@ async function infoForImageFile( return imageInfo; } +/** + * Load a file into a newly created audio element and load the metadata + * + * @param {File} audioFile The file to load in an audio element. + * @return {Promise} A promise that resolves with the audio element. + */ +function loadAudioElement(audioFile: File): Promise { + return new Promise((resolve, reject) => { + // Load the file into a html element + const audio = document.createElement("audio"); + audio.preload = "metadata"; + audio.muted = true; + + const reader = new FileReader(); + + reader.onload = function (ev): void { + audio.onloadedmetadata = async function (): Promise { + resolve(audio); + }; + audio.onerror = function (e): void { + reject(e); + }; + + audio.src = ev.target?.result as string; + }; + reader.onerror = function (e): void { + reject(e); + }; + reader.readAsDataURL(audioFile); + }); +} + +/** + * Read the metadata for an audio file. + * + * @param {File} audioFile The audio to read. + * @return {Promise} A promise that resolves with the attachment info. + */ +async function infoForAudioFile(audioFile: File): Promise { + const audio = await loadAudioElement(audioFile); + return { duration: Math.ceil(audio.duration * 1000) }; +} + /** * Load a file into a newly created video element and pull some strings * in an attempt to guarantee the first frame will be showing. * - * @param {File} videoFile The file to load in an video element. - * @return {Promise} A promise that resolves with the video image element. + * @param {File} videoFile The file to load in a video element. + * @return {Promise} A promise that resolves with the video element. */ function loadVideoElement(videoFile: File): Promise { return new Promise((resolve, reject) => { - // Load the file into an html element + // Load the file into a html element const video = document.createElement("video"); video.preload = "metadata"; video.playsInline = true; @@ -237,20 +283,17 @@ function loadVideoElement(videoFile: File): Promise { * @param {File} videoFile The video to read and thumbnail. * @return {Promise} A promise that resolves with the attachment info. */ -function infoForVideoFile( - matrixClient: MatrixClient, - roomId: string, - videoFile: File, -): Promise> { +function infoForVideoFile(matrixClient: MatrixClient, roomId: string, videoFile: File): Promise { const thumbnailType = "image/jpeg"; - let videoInfo: Partial; + const videoInfo: VideoInfo = {}; return loadVideoElement(videoFile) .then((video) => { + videoInfo.duration = Math.ceil(video.duration * 1000); return createThumbnail(video, video.videoWidth, video.videoHeight, thumbnailType); }) .then((result) => { - videoInfo = result.info; + Object.assign(videoInfo, result.info); return uploadFile(matrixClient, roomId, result.thumbnail); }) .then((result) => { @@ -299,7 +342,7 @@ export async function uploadFile( file: File | Blob, progressHandler?: UploadOpts["progressHandler"], controller?: AbortController, -): Promise<{ url?: string; file?: IEncryptedFile }> { +): Promise<{ url?: string; file?: EncryptedFile }> { const abortController = controller ?? new AbortController(); // If the room is encrypted then encrypt the file before uploading it. @@ -329,7 +372,7 @@ export async function uploadFile( file: { ...encryptResult.info, url, - } as IEncryptedFile, + } as EncryptedFile, }; } else { const { content_uri: url } = await matrixClient.uploadContent(file, { progressHandler, abortController }); @@ -546,6 +589,14 @@ export default class ContentMessages { } } else if (file.type.indexOf("audio/") === 0) { content.msgtype = MsgType.Audio; + try { + const audioInfo = await infoForAudioFile(file); + Object.assign(content.info, audioInfo); + } catch (e) { + // Failed to process audio file, fall back to uploading an m.file + logger.error(e); + content.msgtype = MsgType.File; + } } else if (file.type.indexOf("video/") === 0) { content.msgtype = MsgType.Video; try { diff --git a/src/components/views/messages/MFileBody.tsx b/src/components/views/messages/MFileBody.tsx index a7159049a1..1fc0004fa1 100644 --- a/src/components/views/messages/MFileBody.tsx +++ b/src/components/views/messages/MFileBody.tsx @@ -198,7 +198,7 @@ export default class MFileBody extends React.Component { const isEncrypted = this.props.mediaEventHelper?.media.isEncrypted; const contentUrl = this.getContentUrl(); const contentFileSize = this.content.info ? this.content.info.size : null; - const fileType = this.content.info ? this.content.info.mimetype : "application/octet-stream"; + const fileType = this.content.info?.mimetype ?? "application/octet-stream"; let placeholder: React.ReactNode = null; if (this.props.showGenericPlaceholder) { diff --git a/src/components/views/messages/MImageBody.tsx b/src/components/views/messages/MImageBody.tsx index 91df9c87ac..3a4c23030b 100644 --- a/src/components/views/messages/MImageBody.tsx +++ b/src/components/views/messages/MImageBody.tsx @@ -29,7 +29,7 @@ import SettingsStore from "../../../settings/SettingsStore"; import Spinner from "../elements/Spinner"; import { Media, mediaFromContent } from "../../../customisations/Media"; import { BLURHASH_FIELD, createThumbnail } from "../../../utils/image-media"; -import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent"; +import { ImageContent } from "../../../customisations/models/IMediaEventContent"; import ImageView from "../elements/ImageView"; import { IBodyProps } from "./IBodyProps"; import { ImageSize, suggestedSize as suggestedImageSize } from "../../../settings/enums/ImageSize"; @@ -102,7 +102,7 @@ export default class MImageBody extends React.Component { return; } - const content = this.props.mxEvent.getContent(); + const content = this.props.mxEvent.getContent(); const httpUrl = this.state.contentUrl; if (!httpUrl) return; const params: Omit, "onFinished"> = { @@ -212,7 +212,7 @@ export default class MImageBody extends React.Component { const thumbWidth = 800; const thumbHeight = 600; - const content = this.props.mxEvent.getContent(); + const content = this.props.mxEvent.getContent(); const media = mediaFromContent(content); const info = content.info; @@ -287,7 +287,7 @@ export default class MImageBody extends React.Component { contentUrl = this.getContentUrl(); } - const content = this.props.mxEvent.getContent(); + const content = this.props.mxEvent.getContent(); let isAnimated = mayBeAnimated(content.info?.mimetype); // If there is no included non-animated thumbnail then we will generate our own, we can't depend on the server @@ -317,7 +317,13 @@ export default class MImageBody extends React.Component { } if (isAnimated) { - const thumb = await createThumbnail(img, img.width, img.height, content.info!.mimetype, false); + const thumb = await createThumbnail( + img, + img.width, + img.height, + content.info?.mimetype ?? "image/jpeg", + false, + ); thumbUrl = URL.createObjectURL(thumb.thumbnail); } } catch (error) { @@ -381,7 +387,7 @@ export default class MImageBody extends React.Component { } } - protected getBanner(content: IMediaEventContent): ReactNode { + protected getBanner(content: ImageContent): ReactNode { // Hide it for the threads list & the file panel where we show it as text anyway. if ( [TimelineRenderingType.ThreadsList, TimelineRenderingType.File].includes(this.context.timelineRenderingType) @@ -395,7 +401,7 @@ export default class MImageBody extends React.Component { protected messageContent( contentUrl: string | null, thumbUrl: string | null, - content: IMediaEventContent, + content: ImageContent, forcedHeight?: number, ): ReactNode { if (!thumbUrl) thumbUrl = contentUrl; // fallback @@ -591,7 +597,7 @@ export default class MImageBody extends React.Component { } public render(): React.ReactNode { - const content = this.props.mxEvent.getContent(); + const content = this.props.mxEvent.getContent(); if (this.state.error) { let errorText = _t("Unable to show image due to error"); diff --git a/src/components/views/messages/MImageReplyBody.tsx b/src/components/views/messages/MImageReplyBody.tsx index 68a39d23d6..c6d61a7ba5 100644 --- a/src/components/views/messages/MImageReplyBody.tsx +++ b/src/components/views/messages/MImageReplyBody.tsx @@ -17,7 +17,7 @@ limitations under the License. import React from "react"; import MImageBody from "./MImageBody"; -import { IMediaEventContent } from "../../../customisations/models/IMediaEventContent"; +import { ImageContent } from "../../../customisations/models/IMediaEventContent"; const FORCED_IMAGE_HEIGHT = 44; @@ -35,7 +35,7 @@ export default class MImageReplyBody extends MImageBody { return super.render(); } - const content = this.props.mxEvent.getContent(); + const content = this.props.mxEvent.getContent(); const thumbnail = this.state.contentUrl ? this.messageContent(this.state.contentUrl, this.state.thumbUrl, content, FORCED_IMAGE_HEIGHT) : undefined; diff --git a/src/customisations/models/IMediaEventContent.ts b/src/customisations/models/IMediaEventContent.ts index bd17ba2204..81714000d9 100644 --- a/src/customisations/models/IMediaEventContent.ts +++ b/src/customisations/models/IMediaEventContent.ts @@ -16,8 +16,21 @@ // TODO: These types should be elsewhere. -export interface IEncryptedFile { +import { MsgType } from "matrix-js-sdk/src/matrix"; + +import { BLURHASH_FIELD } from "../../utils/image-media"; + +/** + * @see https://spec.matrix.org/v1.7/client-server-api/#extensions-to-mroommessage-msgtypes + */ +export interface EncryptedFile { + /** + * The URL to the file. + */ url: string; + /** + * A JSON Web Key object. + */ key: { alg: string; key_ops: string[]; // eslint-disable-line camelcase @@ -25,43 +38,204 @@ export interface IEncryptedFile { k: string; ext: boolean; }; + /** + * The 128-bit unique counter block used by AES-CTR, encoded as unpadded base64. + */ iv: string; + /** + * A map from an algorithm name to a hash of the ciphertext, encoded as unpadded base64. + * Clients should support the SHA-256 hash, which uses the key sha256. + */ hashes: { [alg: string]: string }; + /** + * Version of the encrypted attachment's protocol. Must be v2. + */ v: string; } -export interface IMediaEventInfo { - thumbnail_url?: string; // eslint-disable-line camelcase - thumbnail_file?: IEncryptedFile; // eslint-disable-line camelcase - thumbnail_info?: { - // eslint-disable-line camelcase - mimetype: string; - w?: number; - h?: number; - size?: number; - }; - mimetype: string; +interface ThumbnailInfo { + /** + * The mimetype of the image, e.g. image/jpeg. + */ + mimetype?: string; + /** + * The intended display width of the image in pixels. + * This may differ from the intrinsic dimensions of the image file. + */ w?: number; + /** + * The intended display height of the image in pixels. + * This may differ from the intrinsic dimensions of the image file. + */ h?: number; + /** + * Size of the image in bytes. + */ size?: number; } -export interface IMediaEventContent { - msgtype: string; - body?: string; - filename?: string; // `m.file` optional field - url?: string; // required on unencrypted media - file?: IEncryptedFile; // required for *encrypted* media - info?: IMediaEventInfo; +interface BaseInfo { + mimetype?: string; + size?: number; } +/** + * @see https://spec.matrix.org/v1.7/client-server-api/#mfile + */ +export interface FileInfo extends BaseInfo { + /** + * @see https://github.com/matrix-org/matrix-spec-proposals/pull/2448 + */ + [BLURHASH_FIELD]?: string; + /** + * Information on the encrypted thumbnail file, as specified in End-to-end encryption. + * Only present if the thumbnail is encrypted. + * @see https://spec.matrix.org/v1.7/client-server-api/#sending-encrypted-attachments + */ + thumbnail_file?: EncryptedFile; + /** + * Metadata about the image referred to in thumbnail_url. + */ + thumbnail_info?: ThumbnailInfo; + /** + * The URL to the thumbnail of the file. Only present if the thumbnail is unencrypted. + */ + thumbnail_url?: string; +} + +/** + * @see https://spec.matrix.org/v1.7/client-server-api/#mimage + * + */ +export interface ImageInfo extends FileInfo, ThumbnailInfo {} + +/** + * @see https://spec.matrix.org/v1.7/client-server-api/#mimage + */ +export interface AudioInfo extends BaseInfo { + /** + * The duration of the audio in milliseconds. + */ + duration?: number; +} + +/** + * @see https://spec.matrix.org/v1.7/client-server-api/#mvideo + */ +export interface VideoInfo extends AudioInfo, ImageInfo { + /** + * The duration of the video in milliseconds. + */ + duration?: number; +} + +export type IMediaEventInfo = FileInfo | ImageInfo | AudioInfo | VideoInfo; + +interface BaseContent { + /** + * Required if the file is encrypted. Information on the encrypted file, as specified in End-to-end encryption. + * @see https://spec.matrix.org/v1.7/client-server-api/#sending-encrypted-attachments + */ + file?: EncryptedFile; + /** + * Required if the file is unencrypted. The URL (typically mxc:// URI) to the file. + */ + url?: string; +} + +/** + * @see https://spec.matrix.org/v1.7/client-server-api/#mfile + */ +export interface FileContent extends BaseContent { + /** + * A human-readable description of the file. + * This is recommended to be the filename of the original upload. + */ + body: string; + /** + * The original filename of the uploaded file. + */ + filename?: string; + /** + * Information about the file referred to in url. + */ + info?: FileInfo; + /** + * One of: [m.file]. + */ + msgtype: MsgType.File; +} + +/** + * @see https://spec.matrix.org/v1.7/client-server-api/#mimage + */ +export interface ImageContent extends BaseContent { + /** + * A textual representation of the image. + * This could be the alt text of the image, the filename of the image, + * or some kind of content description for accessibility e.g. ‘image attachment’. + */ + body: string; + /** + * Metadata about the image referred to in url. + */ + info?: ImageInfo; + /** + * One of: [m.image]. + */ + msgtype: MsgType.Image; +} + +/** + * @see https://spec.matrix.org/v1.7/client-server-api/#maudio + */ +export interface AudioContent extends BaseContent { + /** + * A description of the audio e.g. ‘Bee Gees - Stayin’ Alive’, + * or some kind of content description for accessibility e.g. ‘audio attachment’. + */ + body: string; + /** + * Metadata for the audio clip referred to in url. + */ + info?: AudioInfo; + /** + * One of: [m.audio]. + */ + msgtype: MsgType.Audio; +} + +/** + * @see https://spec.matrix.org/v1.7/client-server-api/#mvideo + */ +export interface VideoContent extends BaseContent { + /** + * A description of the video e.g. ‘Gangnam style’, + * or some kind of content description for accessibility e.g. ‘video attachment’. + */ + body: string; + /** + * Metadata about the video clip referred to in url. + */ + info?: VideoInfo; + /** + * One of: [m.video]. + */ + msgtype: MsgType.Video; +} + +/** + * Type representing media event contents for `m.room.message` events listed in the Matrix specification + */ +export type IMediaEventContent = FileContent | ImageContent | AudioContent | VideoContent; + export interface IPreparedMedia extends IMediaObject { thumbnail?: IMediaObject; } export interface IMediaObject { mxc: string; - file?: IEncryptedFile; + file?: EncryptedFile; } /** @@ -73,12 +247,17 @@ export interface IMediaObject { */ export function prepEventContentAsMedia(content: Partial): IPreparedMedia { let thumbnail: IMediaObject | undefined; - if (content?.info?.thumbnail_url) { + if (typeof content?.info === "object" && "thumbnail_url" in content.info && content.info.thumbnail_url) { thumbnail = { mxc: content.info.thumbnail_url, file: content.info.thumbnail_file, }; - } else if (content?.info?.thumbnail_file?.url) { + } else if ( + typeof content?.info === "object" && + "thumbnail_file" in content.info && + typeof content?.info?.thumbnail_file === "object" && + content?.info?.thumbnail_file?.url + ) { thumbnail = { mxc: content.info.thumbnail_file.url, file: content.info.thumbnail_file, diff --git a/src/models/RoomUpload.ts b/src/models/RoomUpload.ts index feda356a29..d6e0be4ca9 100644 --- a/src/models/RoomUpload.ts +++ b/src/models/RoomUpload.ts @@ -16,11 +16,11 @@ limitations under the License. import { IEventRelation, UploadProgress } from "matrix-js-sdk/src/matrix"; -import { IEncryptedFile } from "../customisations/models/IMediaEventContent"; +import { EncryptedFile } from "../customisations/models/IMediaEventContent"; export class RoomUpload { public readonly abortController = new AbortController(); - public promise?: Promise<{ url?: string; file?: IEncryptedFile }>; + public promise?: Promise<{ url?: string; file?: EncryptedFile }>; private uploaded = 0; public constructor( diff --git a/src/utils/DecryptFile.ts b/src/utils/DecryptFile.ts index ed0f3953a3..8c9ef1d825 100644 --- a/src/utils/DecryptFile.ts +++ b/src/utils/DecryptFile.ts @@ -19,7 +19,7 @@ import encrypt from "matrix-encrypt-attachment"; import { parseErrorResponse } from "matrix-js-sdk/src/http-api"; import { mediaFromContent } from "../customisations/Media"; -import { IEncryptedFile, IMediaEventInfo } from "../customisations/models/IMediaEventContent"; +import { EncryptedFile, IMediaEventInfo } from "../customisations/models/IMediaEventContent"; import { getBlobSafeMimeType } from "./blobs"; export class DownloadError extends Error { @@ -40,14 +40,14 @@ export class DecryptError extends Error { /** * Decrypt a file attached to a matrix event. - * @param {IEncryptedFile} file The encrypted file information taken from the matrix event. + * @param {EncryptedFile} file The encrypted file information taken from the matrix event. * This passed to [link]{@link https://github.com/matrix-org/matrix-encrypt-attachment} * as the encryption info object, so will also have the those keys in addition to * the keys below. * @param {IMediaEventInfo} info The info parameter taken from the matrix event. * @returns {Promise} Resolves to a Blob of the file. */ -export async function decryptFile(file?: IEncryptedFile, info?: IMediaEventInfo): Promise { +export async function decryptFile(file?: EncryptedFile, info?: IMediaEventInfo): Promise { // throws if file is falsy const media = mediaFromContent({ file }); diff --git a/src/utils/MediaEventHelper.ts b/src/utils/MediaEventHelper.ts index 06bac223cd..f9461b6490 100644 --- a/src/utils/MediaEventHelper.ts +++ b/src/utils/MediaEventHelper.ts @@ -21,7 +21,7 @@ import { logger } from "matrix-js-sdk/src/logger"; import { LazyValue } from "./LazyValue"; import { Media, mediaFromContent } from "../customisations/Media"; import { decryptFile } from "./DecryptFile"; -import { IMediaEventContent } from "../customisations/models/IMediaEventContent"; +import { FileContent, ImageContent, IMediaEventContent } from "../customisations/models/IMediaEventContent"; import { IDestroyable } from "./IDestroyable"; // TODO: We should consider caching the blobs. https://github.com/vector-im/element-web/issues/17192 @@ -48,7 +48,7 @@ export class MediaEventHelper implements IDestroyable { public get fileName(): string { return ( - this.event.getContent().filename || + this.event.getContent().filename || this.event.getContent().body || "download" ); @@ -92,7 +92,7 @@ export class MediaEventHelper implements IDestroyable { if (!this.media.hasThumbnail) return Promise.resolve(null); if (this.media.isEncrypted) { - const content = this.event.getContent(); + const content = this.event.getContent(); if (content.info?.thumbnail_file) { return decryptFile(content.info.thumbnail_file, content.info.thumbnail_info); } else { diff --git a/src/utils/image-media.ts b/src/utils/image-media.ts index b69d552355..fa252ee1ae 100644 --- a/src/utils/image-media.ts +++ b/src/utils/image-media.ts @@ -15,7 +15,7 @@ limitations under the License. */ import { BlurhashEncoder } from "../BlurhashEncoder"; -import { IEncryptedFile } from "../customisations/models/IMediaEventContent"; +import { EncryptedFile } from "../customisations/models/IMediaEventContent"; type ThumbnailableElement = HTMLImageElement | HTMLVideoElement; @@ -33,7 +33,7 @@ interface IThumbnail { h: number; [BLURHASH_FIELD]?: string; thumbnail_url?: string; - thumbnail_file?: IEncryptedFile; + thumbnail_file?: EncryptedFile; }; thumbnail: Blob; } diff --git a/src/voice-broadcast/models/VoiceBroadcastRecording.ts b/src/voice-broadcast/models/VoiceBroadcastRecording.ts index 53ab42a185..db6a14d4e6 100644 --- a/src/voice-broadcast/models/VoiceBroadcastRecording.ts +++ b/src/voice-broadcast/models/VoiceBroadcastRecording.ts @@ -38,7 +38,7 @@ import { VoiceBroadcastRecorderEvent, } from ".."; import { uploadFile } from "../../ContentMessages"; -import { IEncryptedFile } from "../../customisations/models/IMediaEventContent"; +import { EncryptedFile } from "../../customisations/models/IMediaEventContent"; import { createVoiceMessageContent } from "../../utils/createVoiceMessageContent"; import { IDestroyable } from "../../utils/IDestroyable"; import dis from "../../dispatcher/dispatcher"; @@ -367,7 +367,7 @@ export class VoiceBroadcastRecording ); } - private async sendVoiceMessage(chunk: ChunkRecordedPayload, url?: string, file?: IEncryptedFile): Promise { + private async sendVoiceMessage(chunk: ChunkRecordedPayload, url?: string, file?: EncryptedFile): Promise { /** * Increment the last sequence number and use it for this message. * Done outside of the sendMessageFn to get a scoped value. diff --git a/test/ContentMessages-test.ts b/test/ContentMessages-test.ts index 64bd3c845c..d1a4ecf72d 100644 --- a/test/ContentMessages-test.ts +++ b/test/ContentMessages-test.ts @@ -163,6 +163,11 @@ describe("ContentMessages", () => { return 800; }, }); + Object.defineProperty(element, "duration", { + get() { + return 123; + }, + }); } return element; }); @@ -176,11 +181,31 @@ describe("ContentMessages", () => { expect.objectContaining({ url: "mxc://server/file", msgtype: "m.video", + info: expect.objectContaining({ + duration: 123000, + }), }), ); }); it("should use m.audio for audio files", async () => { + jest.spyOn(document, "createElement").mockImplementation((tagName) => { + const element = createElement(tagName); + if (tagName === "audio") { + Object.defineProperty(element, "duration", { + get() { + return 621; + }, + }); + Object.defineProperty(element, "src", { + set() { + element.onloadedmetadata!(new Event("loadedmetadata")); + }, + }); + } + return element; + }); + mocked(client.uploadContent).mockResolvedValue({ content_uri: "mxc://server/file" }); const file = new File([], "fileName", { type: "audio/mp3" }); await contentMessages.sendContentToRoom(file, roomId, undefined, client, undefined); @@ -190,6 +215,34 @@ describe("ContentMessages", () => { expect.objectContaining({ url: "mxc://server/file", msgtype: "m.audio", + info: expect.objectContaining({ + duration: 621000, + }), + }), + ); + }); + + it("should fall back to m.file for invalid audio files", async () => { + jest.spyOn(document, "createElement").mockImplementation((tagName) => { + const element = createElement(tagName); + if (tagName === "audio") { + Object.defineProperty(element, "src", { + set() { + element.onerror!("fail"); + }, + }); + } + return element; + }); + mocked(client.uploadContent).mockResolvedValue({ content_uri: "mxc://server/file" }); + const file = new File([], "fileName", { type: "audio/mp3" }); + await contentMessages.sendContentToRoom(file, roomId, undefined, client, undefined); + expect(client.sendMessage).toHaveBeenCalledWith( + roomId, + null, + expect.objectContaining({ + url: "mxc://server/file", + msgtype: "m.file", }), ); }); diff --git a/test/components/views/messages/MImageBody-test.tsx b/test/components/views/messages/MImageBody-test.tsx index 71b3a79dcc..8af6ea96d8 100644 --- a/test/components/views/messages/MImageBody-test.tsx +++ b/test/components/views/messages/MImageBody-test.tsx @@ -15,11 +15,13 @@ limitations under the License. */ import React from "react"; -import { fireEvent, render, screen } from "@testing-library/react"; +import { fireEvent, render, screen, waitForElementToBeRemoved } from "@testing-library/react"; import { EventType, getHttpUriForMxc, MatrixEvent, Room } from "matrix-js-sdk/src/matrix"; import fetchMock from "fetch-mock-jest"; import encrypt from "matrix-encrypt-attachment"; import { mocked } from "jest-mock"; +import fs from "fs"; +import path from "path"; import MImageBody from "../../../../src/components/views/messages/MImageBody"; import { RoomPermalinkCreator } from "../../../../src/utils/permalinks/Permalinks"; @@ -56,7 +58,11 @@ describe("", () => { }); const url = "https://server/_matrix/media/r0/download/server/encrypted-image"; // eslint-disable-next-line no-restricted-properties - cli.mxcUrlToHttp.mockReturnValue(url); + cli.mxcUrlToHttp.mockImplementation( + (mxcUrl: string, width?: number, height?: number, resizeMethod?: string, allowDirectLinks?: boolean) => { + return getHttpUriForMxc("https://server", mxcUrl, width, height, resizeMethod, allowDirectLinks); + }, + ); const encryptedMediaEvent = new MatrixEvent({ room_id: "!room:server", sender: userId, @@ -175,12 +181,6 @@ describe("", () => { it("should fall back to /download/ if /thumbnail/ fails", async () => { const thumbUrl = "https://server/_matrix/media/r0/thumbnail/server/image?width=800&height=600&method=scale"; const downloadUrl = "https://server/_matrix/media/r0/download/server/image"; - // eslint-disable-next-line no-restricted-properties - cli.mxcUrlToHttp.mockImplementation( - (mxcUrl: string, width?: number, height?: number, resizeMethod?: string, allowDirectLinks?: boolean) => { - return getHttpUriForMxc("https://server", mxcUrl, width, height, resizeMethod, allowDirectLinks); - }, - ); const event = new MatrixEvent({ room_id: "!room:server", @@ -206,4 +206,56 @@ describe("", () => { fireEvent.error(img); expect(img).toHaveProperty("src", downloadUrl); }); + + it("should generate a thumbnail if one isn't included for animated media", async () => { + Object.defineProperty(global.Image.prototype, "src", { + set(src) { + window.setTimeout(() => this.onload()); + }, + }); + Object.defineProperty(global.Image.prototype, "height", { + get() { + return 600; + }, + }); + Object.defineProperty(global.Image.prototype, "width", { + get() { + return 800; + }, + }); + + mocked(global.URL.createObjectURL).mockReturnValue("blob:generated-thumb"); + + fetchMock.getOnce( + "https://server/_matrix/media/r0/download/server/image", + { + body: fs.readFileSync(path.resolve(__dirname, "..", "..", "..", "images", "animated-logo.webp")), + }, + { sendAsJson: false }, + ); + + const event = new MatrixEvent({ + room_id: "!room:server", + sender: userId, + type: EventType.RoomMessage, + content: { + body: "alt for a test image", + info: { + w: 40, + h: 50, + mimetype: "image/webp", + }, + url: "mxc://server/image", + }, + }); + + const { container } = render( + , + ); + + // Wait for spinners to go away + await waitForElementToBeRemoved(screen.getAllByRole("progressbar")); + // thumbnail with dimensions present + expect(container).toMatchSnapshot(); + }); }); diff --git a/test/components/views/messages/MVideoBody-test.tsx b/test/components/views/messages/MVideoBody-test.tsx index 087c736a9b..49263046e9 100644 --- a/test/components/views/messages/MVideoBody-test.tsx +++ b/test/components/views/messages/MVideoBody-test.tsx @@ -15,23 +15,22 @@ limitations under the License. */ import React, { ComponentProps } from "react"; -import { IContent, MatrixEvent } from "matrix-js-sdk/src/matrix"; +import { EventType, getHttpUriForMxc, IContent, MatrixEvent } from "matrix-js-sdk/src/matrix"; import { render, RenderResult } from "@testing-library/react"; +import fetchMock from "fetch-mock-jest"; import MatrixClientContext from "../../../../src/contexts/MatrixClientContext"; import { RoomPermalinkCreator } from "../../../../src/utils/permalinks/Permalinks"; import { MediaEventHelper } from "../../../../src/utils/MediaEventHelper"; -import { getMockClientWithEventEmitter } from "../../../test-utils"; +import { + getMockClientWithEventEmitter, + mockClientMethodsCrypto, + mockClientMethodsDevice, + mockClientMethodsServer, + mockClientMethodsUser, +} from "../../../test-utils"; import MVideoBody from "../../../../src/components/views/messages/MVideoBody"; -jest.mock("../../../../src/customisations/Media", () => { - return { - mediaFromContent: () => { - return { isEncrypted: false }; - }, - }; -}); - describe("MVideoBody", () => { it("does not crash when given a portrait image", () => { // Check for an unreliable crash caused by a fractional-sized @@ -40,6 +39,58 @@ describe("MVideoBody", () => { expect(asFragment()).toMatchSnapshot(); // If we get here, we did not crash. }); + + it("should show poster for encrypted media before downloading it", async () => { + const userId = "@user:server"; + const deviceId = "DEADB33F"; + const cli = getMockClientWithEventEmitter({ + ...mockClientMethodsUser(userId), + ...mockClientMethodsServer(), + ...mockClientMethodsDevice(deviceId), + ...mockClientMethodsCrypto(), + getRooms: jest.fn().mockReturnValue([]), + getIgnoredUsers: jest.fn(), + getVersions: jest.fn().mockResolvedValue({ + unstable_features: { + "org.matrix.msc3882": true, + "org.matrix.msc3886": true, + }, + }), + }); + const thumbUrl = "https://server/_matrix/media/r0/download/server/encrypted-poster"; + fetchMock.getOnce(thumbUrl, { status: 200 }); + + // eslint-disable-next-line no-restricted-properties + cli.mxcUrlToHttp.mockImplementation( + (mxcUrl: string, width?: number, height?: number, resizeMethod?: string, allowDirectLinks?: boolean) => { + return getHttpUriForMxc("https://server", mxcUrl, width, height, resizeMethod, allowDirectLinks); + }, + ); + const encryptedMediaEvent = new MatrixEvent({ + room_id: "!room:server", + sender: userId, + type: EventType.RoomMessage, + content: { + body: "alt for a test video", + info: { + duration: 420, + w: 40, + h: 50, + thumbnail_file: { + url: "mxc://server/encrypted-poster", + }, + }, + file: { + url: "mxc://server/encrypted-image", + }, + }, + }); + + const { asFragment } = render( + , + ); + expect(asFragment()).toMatchSnapshot(); + }); }); function makeMVideoBody(w: number, h: number): RenderResult { diff --git a/test/components/views/messages/__snapshots__/MImageBody-test.tsx.snap b/test/components/views/messages/__snapshots__/MImageBody-test.tsx.snap index 553090de62..b38b53e937 100644 --- a/test/components/views/messages/__snapshots__/MImageBody-test.tsx.snap +++ b/test/components/views/messages/__snapshots__/MImageBody-test.tsx.snap @@ -1,5 +1,55 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[` should generate a thumbnail if one isn't included for animated media 1`] = ` +
+
+ +
+
+
+
+
+
+
+ alt for a test image +

+ GIF +

+
+
+
+ +
+
+`; + exports[` should show a thumbnail while image is being downloaded 1`] = `
`; + +exports[`MVideoBody should show poster for encrypted media before downloading it 1`] = ` + + +
+