feat: image support (#4011)

Co-authored-by: Emil Atanasov <heitara@gmail.com>
Co-authored-by: Aakansha Doshi <aakansha1216@gmail.com>
This commit is contained in:
David Luzar
2021-10-21 22:05:48 +02:00
committed by GitHub
parent 0f0244224d
commit 163ad1f4c4
85 changed files with 3536 additions and 618 deletions

View File

@ -0,0 +1,249 @@
import { compressData } from "../../data/encode";
import { mutateElement } from "../../element/mutateElement";
import { isInitializedImageElement } from "../../element/typeChecks";
import {
ExcalidrawElement,
ExcalidrawImageElement,
FileId,
InitializedExcalidrawImageElement,
} from "../../element/types";
import { t } from "../../i18n";
import {
BinaryFileData,
BinaryFileMetadata,
ExcalidrawImperativeAPI,
BinaryFiles,
} from "../../types";
export class FileManager {
/** files being fetched */
private fetchingFiles = new Map<ExcalidrawImageElement["fileId"], true>();
/** files being saved */
private savingFiles = new Map<ExcalidrawImageElement["fileId"], true>();
/* files already saved to persistent storage */
private savedFiles = new Map<ExcalidrawImageElement["fileId"], true>();
private erroredFiles = new Map<ExcalidrawImageElement["fileId"], true>();
private _getFiles;
private _saveFiles;
constructor({
getFiles,
saveFiles,
}: {
getFiles: (
fileIds: FileId[],
) => Promise<{
loadedFiles: BinaryFileData[];
erroredFiles: Map<FileId, true>;
}>;
saveFiles: (data: {
addedFiles: Map<FileId, BinaryFileData>;
}) => Promise<{
savedFiles: Map<FileId, true>;
erroredFiles: Map<FileId, true>;
}>;
}) {
this._getFiles = getFiles;
this._saveFiles = saveFiles;
}
/**
* returns whether file is already saved or being processed
*/
isFileHandled = (id: FileId) => {
return (
this.savedFiles.has(id) ||
this.fetchingFiles.has(id) ||
this.savingFiles.has(id) ||
this.erroredFiles.has(id)
);
};
isFileSaved = (id: FileId) => {
return this.savedFiles.has(id);
};
saveFiles = async ({
elements,
files,
}: {
elements: readonly ExcalidrawElement[];
files: BinaryFiles;
}) => {
const addedFiles: Map<FileId, BinaryFileData> = new Map();
for (const element of elements) {
if (
isInitializedImageElement(element) &&
files[element.fileId] &&
!this.isFileHandled(element.fileId)
) {
addedFiles.set(element.fileId, files[element.fileId]);
this.savingFiles.set(element.fileId, true);
}
}
try {
const { savedFiles, erroredFiles } = await this._saveFiles({
addedFiles,
});
for (const [fileId] of savedFiles) {
this.savedFiles.set(fileId, true);
}
return {
savedFiles,
erroredFiles,
};
} finally {
for (const [fileId] of addedFiles) {
this.savingFiles.delete(fileId);
}
}
};
getFiles = async (
ids: FileId[],
): Promise<{
loadedFiles: BinaryFileData[];
erroredFiles: Map<FileId, true>;
}> => {
if (!ids.length) {
return {
loadedFiles: [],
erroredFiles: new Map(),
};
}
for (const id of ids) {
this.fetchingFiles.set(id, true);
}
try {
const { loadedFiles, erroredFiles } = await this._getFiles(ids);
for (const file of loadedFiles) {
this.savedFiles.set(file.id, true);
}
for (const [fileId] of erroredFiles) {
this.erroredFiles.set(fileId, true);
}
return { loadedFiles, erroredFiles };
} finally {
for (const id of ids) {
this.fetchingFiles.delete(id);
}
}
};
/** a file element prevents unload only if it's being saved regardless of
* its `status`. This ensures that elements who for any reason haven't
* beed set to `saved` status don't prevent unload in future sessions.
* Technically we should prevent unload when the origin client haven't
* yet saved the `status` update to storage, but that should be taken care
* of during regular beforeUnload unsaved files check.
*/
shouldPreventUnload = (elements: readonly ExcalidrawElement[]) => {
return elements.some((element) => {
return (
isInitializedImageElement(element) &&
!element.isDeleted &&
this.savingFiles.has(element.fileId)
);
});
};
/**
* helper to determine if image element status needs updating
*/
shouldUpdateImageElementStatus = (
element: ExcalidrawElement,
): element is InitializedExcalidrawImageElement => {
return (
isInitializedImageElement(element) &&
this.isFileSaved(element.fileId) &&
element.status === "pending"
);
};
reset() {
this.fetchingFiles.clear();
this.savingFiles.clear();
this.savedFiles.clear();
this.erroredFiles.clear();
}
}
export const encodeFilesForUpload = async ({
files,
maxBytes,
encryptionKey,
}: {
files: Map<FileId, BinaryFileData>;
maxBytes: number;
encryptionKey: string;
}) => {
const processedFiles: {
id: FileId;
buffer: Uint8Array;
}[] = [];
for (const [id, fileData] of files) {
const buffer = new TextEncoder().encode(fileData.dataURL);
const encodedFile = await compressData<BinaryFileMetadata>(buffer, {
encryptionKey,
metadata: {
id,
mimeType: fileData.mimeType,
created: Date.now(),
},
});
if (buffer.byteLength > maxBytes) {
throw new Error(
t("errors.fileTooBig", {
maxSize: `${Math.trunc(maxBytes / 1024 / 1024)}MB`,
}),
);
}
processedFiles.push({
id,
buffer: encodedFile,
});
}
return processedFiles;
};
export const updateStaleImageStatuses = (params: {
excalidrawAPI: ExcalidrawImperativeAPI;
erroredFiles: Map<FileId, true>;
elements: readonly ExcalidrawElement[];
}) => {
if (!params.erroredFiles.size) {
return;
}
params.excalidrawAPI.updateScene({
elements: params.excalidrawAPI
.getSceneElementsIncludingDeleted()
.map((element) => {
if (
isInitializedImageElement(element) &&
params.erroredFiles.has(element.fileId)
) {
return mutateElement(
element,
{
status: "error",
},
false,
);
}
return element;
}),
});
};

View File

@ -1,26 +1,45 @@
import { getImportedKey } from "../data";
import { createIV } from "./index";
import { ExcalidrawElement } from "../../element/types";
import { ExcalidrawElement, FileId } from "../../element/types";
import { getSceneVersion } from "../../element";
import Portal from "../collab/Portal";
import { restoreElements } from "../../data/restore";
import { BinaryFileData, BinaryFileMetadata, DataURL } from "../../types";
import { FILE_CACHE_MAX_AGE_SEC } from "../app_constants";
import { decompressData } from "../../data/encode";
import { getImportedKey, createIV } from "../../data/encryption";
import { MIME_TYPES } from "../../constants";
// private
// -----------------------------------------------------------------------------
const FIREBASE_CONFIG = JSON.parse(process.env.REACT_APP_FIREBASE_CONFIG);
let firebasePromise: Promise<
typeof import("firebase/app").default
> | null = null;
let firestorePromise: Promise<any> | null = null;
let firebseStoragePromise: Promise<any> | null = null;
let firestorePromise: Promise<any> | null | true = null;
let firebaseStoragePromise: Promise<any> | null | true = null;
let isFirebaseInitialized = false;
const _loadFirebase = async () => {
const firebase = (
await import(/* webpackChunkName: "firebase" */ "firebase/app")
).default;
const firebaseConfig = JSON.parse(process.env.REACT_APP_FIREBASE_CONFIG);
firebase.initializeApp(firebaseConfig);
if (!isFirebaseInitialized) {
try {
firebase.initializeApp(FIREBASE_CONFIG);
} catch (error) {
// trying initialize again throws. Usually this is harmless, and happens
// mainly in dev (HMR)
if (error.code === "app/duplicate-app") {
console.warn(error.name, error.code);
} else {
throw error;
}
}
isFirebaseInitialized = true;
}
return firebase;
};
@ -42,18 +61,24 @@ const loadFirestore = async () => {
firestorePromise = import(
/* webpackChunkName: "firestore" */ "firebase/firestore"
);
}
if (firestorePromise !== true) {
await firestorePromise;
firestorePromise = true;
}
return firebase;
};
export const loadFirebaseStorage = async () => {
const firebase = await _getFirebase();
if (!firebseStoragePromise) {
firebseStoragePromise = import(
if (!firebaseStoragePromise) {
firebaseStoragePromise = import(
/* webpackChunkName: "storage" */ "firebase/storage"
);
await firebseStoragePromise;
}
if (firebaseStoragePromise !== true) {
await firebaseStoragePromise;
firebaseStoragePromise = true;
}
return firebase;
};
@ -87,7 +112,7 @@ const encryptElements = async (
const decryptElements = async (
key: string,
iv: Uint8Array,
ciphertext: ArrayBuffer,
ciphertext: ArrayBuffer | Uint8Array,
): Promise<readonly ExcalidrawElement[]> => {
const importedKey = await getImportedKey(key, "decrypt");
const decrypted = await window.crypto.subtle.decrypt(
@ -100,7 +125,7 @@ const decryptElements = async (
);
const decodedData = new TextDecoder("utf-8").decode(
new Uint8Array(decrypted) as any,
new Uint8Array(decrypted),
);
return JSON.parse(decodedData);
};
@ -113,6 +138,7 @@ export const isSavedToFirebase = (
): boolean => {
if (portal.socket && portal.roomId && portal.roomKey) {
const sceneVersion = getSceneVersion(elements);
return firebaseSceneVersionCache.get(portal.socket) === sceneVersion;
}
// if no room exists, consider the room saved so that we don't unnecessarily
@ -120,6 +146,42 @@ export const isSavedToFirebase = (
return true;
};
export const saveFilesToFirebase = async ({
prefix,
files,
}: {
prefix: string;
files: { id: FileId; buffer: Uint8Array }[];
}) => {
const firebase = await loadFirebaseStorage();
const erroredFiles = new Map<FileId, true>();
const savedFiles = new Map<FileId, true>();
await Promise.all(
files.map(async ({ id, buffer }) => {
try {
await firebase
.storage()
.ref(`${prefix}/${id}`)
.put(
new Blob([buffer], {
type: MIME_TYPES.binary,
}),
{
cacheControl: `public, max-age=${FILE_CACHE_MAX_AGE_SEC}`,
},
);
savedFiles.set(id, true);
} catch (error) {
erroredFiles.set(id, true);
}
}),
);
return { savedFiles, erroredFiles };
};
export const saveToFirebase = async (
portal: Portal,
elements: readonly ExcalidrawElement[],
@ -198,3 +260,47 @@ export const loadFromFirebase = async (
return restoreElements(elements, null);
};
export const loadFilesFromFirebase = async (
prefix: string,
decryptionKey: string,
filesIds: readonly FileId[],
) => {
const loadedFiles: BinaryFileData[] = [];
const erroredFiles = new Map<FileId, true>();
await Promise.all(
[...new Set(filesIds)].map(async (id) => {
try {
const url = `https://firebasestorage.googleapis.com/v0/b/${
FIREBASE_CONFIG.storageBucket
}/o/${encodeURIComponent(prefix.replace(/^\//, ""))}%2F${id}`;
const response = await fetch(`${url}?alt=media`);
if (response.status < 400) {
const arrayBuffer = await response.arrayBuffer();
const { data, metadata } = await decompressData<BinaryFileMetadata>(
new Uint8Array(arrayBuffer),
{
decryptionKey,
},
);
const dataURL = new TextDecoder().decode(data) as DataURL;
loadedFiles.push({
mimeType: metadata.mimeType || MIME_TYPES.binary,
id,
dataURL,
created: metadata?.created || Date.now(),
});
}
} catch (error) {
erroredFiles.set(id, true);
console.error(error);
}
}),
);
return { loadedFiles, erroredFiles };
};

View File

@ -1,9 +1,24 @@
import {
createIV,
generateEncryptionKey,
getImportedKey,
IV_LENGTH_BYTES,
} from "../../data/encryption";
import { serializeAsJSON } from "../../data/json";
import { restore } from "../../data/restore";
import { ImportedDataState } from "../../data/types";
import { ExcalidrawElement } from "../../element/types";
import { isInitializedImageElement } from "../../element/typeChecks";
import { ExcalidrawElement, FileId } from "../../element/types";
import { t } from "../../i18n";
import { AppState, UserIdleState } from "../../types";
import {
AppState,
BinaryFileData,
BinaryFiles,
UserIdleState,
} from "../../types";
import { FILE_UPLOAD_MAX_BYTES } from "../app_constants";
import { encodeFilesForUpload } from "./FileManager";
import { saveFilesToFirebase } from "./firebase";
const byteToHex = (byte: number): string => `0${byte.toString(16)}`.slice(-2);
@ -17,18 +32,6 @@ const generateRandomID = async () => {
return Array.from(arr, byteToHex).join("");
};
export const generateEncryptionKey = async () => {
const key = await window.crypto.subtle.generateKey(
{
name: "AES-GCM",
length: 128,
},
true, // extractable
["encrypt", "decrypt"],
);
return (await window.crypto.subtle.exportKey("jwk", key)).k;
};
export const SOCKET_SERVER = process.env.REACT_APP_SOCKET_SERVER_URL;
export type EncryptedData = {
@ -79,13 +82,6 @@ export type SocketUpdateData = SocketUpdateDataSource[keyof SocketUpdateDataSour
_brand: "socketUpdateData";
};
const IV_LENGTH_BYTES = 12; // 96 bits
export const createIV = () => {
const arr = new Uint8Array(IV_LENGTH_BYTES);
return window.crypto.getRandomValues(arr);
};
export const encryptAESGEM = async (
data: Uint8Array,
key: string,
@ -122,7 +118,7 @@ export const decryptAESGEM = async (
);
const decodedData = new TextDecoder("utf-8").decode(
new Uint8Array(decrypted) as any,
new Uint8Array(decrypted),
);
return JSON.parse(decodedData);
} catch (error) {
@ -162,26 +158,8 @@ export const getCollaborationLink = (data: {
return `${window.location.origin}${window.location.pathname}#room=${data.roomId},${data.roomKey}`;
};
export const getImportedKey = (key: string, usage: KeyUsage) =>
window.crypto.subtle.importKey(
"jwk",
{
alg: "A128GCM",
ext: true,
k: key,
key_ops: ["encrypt", "decrypt"],
kty: "oct",
},
{
name: "AES-GCM",
length: 128,
},
false, // extractable
[usage],
);
export const decryptImported = async (
iv: ArrayBuffer,
iv: ArrayBuffer | Uint8Array,
encrypted: ArrayBuffer,
privateKey: string,
): Promise<ArrayBuffer> => {
@ -227,7 +205,7 @@ const importFromBackend = async (
// We need to convert the decrypted array buffer to a string
const string = new window.TextDecoder("utf-8").decode(
new Uint8Array(decrypted) as any,
new Uint8Array(decrypted),
);
data = JSON.parse(string);
} else {
@ -270,6 +248,10 @@ export const loadScene = async (
return {
elements: data.elements,
appState: data.appState,
// note: this will always be empty because we're not storing files
// in the scene database/localStorage, and instead fetch them async
// from a different database
files: data.files,
commitToHistory: false,
};
};
@ -277,11 +259,12 @@ export const loadScene = async (
export const exportToBackend = async (
elements: readonly ExcalidrawElement[],
appState: AppState,
files: BinaryFiles,
) => {
const json = serializeAsJSON(elements, appState);
const json = serializeAsJSON(elements, appState, files, "database");
const encoded = new TextEncoder().encode(json);
const key = await window.crypto.subtle.generateKey(
const cryptoKey = await window.crypto.subtle.generateKey(
{
name: "AES-GCM",
length: 128,
@ -298,7 +281,7 @@ export const exportToBackend = async (
name: "AES-GCM",
iv,
},
key,
cryptoKey,
encoded,
);
@ -308,9 +291,24 @@ export const exportToBackend = async (
// We use jwk encoding to be able to extract just the base64 encoded key.
// We will hardcode the rest of the attributes when importing back the key.
const exportedKey = await window.crypto.subtle.exportKey("jwk", key);
const exportedKey = await window.crypto.subtle.exportKey("jwk", cryptoKey);
try {
const filesMap = new Map<FileId, BinaryFileData>();
for (const element of elements) {
if (isInitializedImageElement(element) && files[element.fileId]) {
filesMap.set(element.fileId, files[element.fileId]);
}
}
const encryptionKey = exportedKey.k!;
const filesToUpload = await encodeFilesForUpload({
files: filesMap,
encryptionKey,
maxBytes: FILE_UPLOAD_MAX_BYTES,
});
const response = await fetch(BACKEND_V2_POST, {
method: "POST",
body: payload,
@ -320,8 +318,14 @@ export const exportToBackend = async (
const url = new URL(window.location.href);
// We need to store the key (and less importantly the id) as hash instead
// of queryParam in order to never send it to the server
url.hash = `json=${json.id},${exportedKey.k!}`;
url.hash = `json=${json.id},${encryptionKey}`;
const urlString = url.toString();
await saveFilesToFirebase({
prefix: `/files/shareLinks/${json.id}`,
files: filesToUpload,
});
window.prompt(`🔒${t("alerts.uploadedSecurly")}`, urlString);
} else if (json.error_class === "RequestTooLargeError") {
window.alert(t("alerts.couldNotCreateShareableLinkTooBig"));