From ad571b12b898aae9f117bce13035d926cf373fb3 Mon Sep 17 00:00:00 2001 From: Jannik Streek Date: Mon, 1 Jan 2024 18:05:48 +0100 Subject: [PATCH] removed firebase and added support for rooms without firebase and fix docker setup --- .dockerignore | 3 +- .env.default | 1 + .env.development | 15 +- .env.production | 18 +- .gitignore | 1 + Dockerfile | 17 +- README.md | 79 ++++++-- docker-compose-prod.yml | 44 +++++ docker-compose.yml | 37 +++- excalidraw-app/App.tsx | 15 +- excalidraw-app/collab/Collab.tsx | 29 +-- excalidraw-app/collab/Portal.tsx | 4 +- excalidraw-app/data/StorageBackend.ts | 45 +++++ excalidraw-app/data/config.ts | 54 ++++++ excalidraw-app/data/firebase.ts | 14 +- excalidraw-app/data/httpStorage.ts | 266 ++++++++++++++++++++++++++ excalidraw-app/data/index.ts | 6 +- excalidraw-app/package.json | 2 +- excalidraw-app/vite-env.d.ts | 3 + excalidraw-app/vite.config.mts | 5 +- package.json | 7 +- 21 files changed, 585 insertions(+), 80 deletions(-) create mode 100644 .env.default create mode 100644 docker-compose-prod.yml create mode 100644 excalidraw-app/data/StorageBackend.ts create mode 100644 excalidraw-app/data/config.ts create mode 100644 excalidraw-app/data/httpStorage.ts diff --git a/.dockerignore b/.dockerignore index 1f38a978..98c9151f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,8 +4,9 @@ !.eslintrc.json !.npmrc !.prettierrc +!excalidraw-app/ !package.json !public/ !packages/ !tsconfig.json -!yarn.lock +!yarn.lock \ No newline at end of file diff --git a/.env.default b/.env.default new file mode 100644 index 00000000..531584cf --- /dev/null +++ b/.env.default @@ -0,0 +1 @@ +REDIS_PASSWORD=CHANGE_ME \ No newline at end of file diff --git a/.env.development b/.env.development index 44955884..bf7a32fd 100644 --- a/.env.development +++ b/.env.development @@ -1,11 +1,14 @@ -VITE_APP_BACKEND_V2_GET_URL=https://json-dev.excalidraw.com/api/v2/ -VITE_APP_BACKEND_V2_POST_URL=https://json-dev.excalidraw.com/api/v2/post/ +VITE_APP_BACKEND_V2_GET_URL=http://localhost:8080/api/v2/scenes/ +VITE_APP_BACKEND_V2_POST_URL=http://localhost:8080/api/v2/scenes/ VITE_APP_LIBRARY_URL=https://libraries.excalidraw.com VITE_APP_LIBRARY_BACKEND=https://us-central1-excalidraw-room-persistence.cloudfunctions.net/libraries +VITE_APP_STORAGE_BACKEND=http +VITE_APP_HTTP_STORAGE_BACKEND_URL=http://localhost:8080/api/v2 + # collaboration WebSocket server (https://github.com/excalidraw/excalidraw-room) -VITE_APP_WS_SERVER_URL=http://localhost:3002 +VITE_APP_WS_SERVER_URL=http://localhost:5001 # set this only if using the collaboration workflow we use on excalidraw.com VITE_APP_PORTAL_URL= @@ -13,9 +16,9 @@ VITE_APP_PORTAL_URL= VITE_APP_PLUS_LP=https://plus.excalidraw.com VITE_APP_PLUS_APP=https://app.excalidraw.com -VITE_APP_AI_BACKEND=http://localhost:3015 +VITE_APP_AI_BACKEND= -VITE_APP_FIREBASE_CONFIG='{"apiKey":"AIzaSyCMkxA60XIW8KbqMYL7edC4qT5l4qHX2h8","authDomain":"excalidraw-oss-dev.firebaseapp.com","projectId":"excalidraw-oss-dev","storageBucket":"excalidraw-oss-dev.appspot.com","messagingSenderId":"664559512677","appId":"1:664559512677:web:a385181f2928d328a7aa8c"}' +# VITE_APP_FIREBASE_CONFIG='{"apiKey":"AIzaSyCMkxA60XIW8KbqMYL7edC4qT5l4qHX2h8","authDomain":"excalidraw-oss-dev.firebaseapp.com","projectId":"excalidraw-oss-dev","storageBucket":"excalidraw-oss-dev.appspot.com","messagingSenderId":"664559512677","appId":"1:664559512677:web:a385181f2928d328a7aa8c"}' # put these in your .env.local, or make sure you don't commit! # must be lowercase `true` when turned on @@ -41,4 +44,4 @@ VITE_APP_DEBUG_ENABLE_TEXT_CONTAINER_BOUNDING_BOX= VITE_APP_COLLAPSE_OVERLAY=true # Set this flag to false to disable eslint -VITE_APP_ENABLE_ESLINT=true +VITE_APP_ENABLE_ESLINT=true \ No newline at end of file diff --git a/.env.production b/.env.production index 26b46a52..a022a78d 100644 --- a/.env.production +++ b/.env.production @@ -1,20 +1,24 @@ -VITE_APP_BACKEND_V2_GET_URL=https://json.excalidraw.com/api/v2/ -VITE_APP_BACKEND_V2_POST_URL=https://json.excalidraw.com/api/v2/post/ +VITE_APP_BACKEND_V2_GET_URL=http://localhost:8080/api/v2/scenes/ +VITE_APP_BACKEND_V2_POST_URL=http://localhost:8080/api/v2/scenes/ VITE_APP_LIBRARY_URL=https://libraries.excalidraw.com VITE_APP_LIBRARY_BACKEND=https://us-central1-excalidraw-room-persistence.cloudfunctions.net/libraries -VITE_APP_PORTAL_URL=https://portal.excalidraw.com +VITE_APP_STORAGE_BACKEND=http +VITE_APP_HTTP_STORAGE_BACKEND_URL=http://localhost:5011/api/v2 + +VITE_APP_PORTAL_URL= VITE_APP_PLUS_LP=https://plus.excalidraw.com VITE_APP_PLUS_APP=https://app.excalidraw.com -VITE_APP_AI_BACKEND=https://oss-ai.excalidraw.com +VITE_APP_AI_BACKEND= # Fill to set socket server URL used for collaboration. # Meant for forks only: excalidraw.com uses custom VITE_APP_PORTAL_URL flow -VITE_APP_WS_SERVER_URL= +VITE_APP_WS_SERVER_URL=http://localhost:5012 -VITE_APP_FIREBASE_CONFIG='{"apiKey":"AIzaSyAd15pYlMci_xIp9ko6wkEsDzAAA0Dn0RU","authDomain":"excalidraw-room-persistence.firebaseapp.com","databaseURL":"https://excalidraw-room-persistence.firebaseio.com","projectId":"excalidraw-room-persistence","storageBucket":"excalidraw-room-persistence.appspot.com","messagingSenderId":"654800341332","appId":"1:654800341332:web:4a692de832b55bd57ce0c1"}' +# VITE_APP_FIREBASE_CONFIG='{"apiKey":"AIzaSyAd15pYlMci_xIp9ko6wkEsDzAAA0Dn0RU","authDomain":"excalidraw-room-persistence.firebaseapp.com","databaseURL":"https://excalidraw-room-persistence.firebaseio.com","projectId":"excalidraw-room-persistence","storageBucket":"excalidraw-room-persistence.appspot.com","messagingSenderId":"654800341332","appId":"1:654800341332:web:4a692de832b55bd57ce0c1"}' -VITE_APP_DISABLE_TRACKING= +VITE_APP_DISABLE_TRACKING=true +VITE_APP_DISABLE_SENTRY=true \ No newline at end of file diff --git a/.gitignore b/.gitignore index d670c78a..fb0393b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .DS_Store .env.development.local .env.local +.env .env.production.local .env.test.local .envrc diff --git a/Dockerfile b/Dockerfile index a044f40f..ed9afaf1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,16 +2,23 @@ FROM node:18 AS build WORKDIR /opt/node_app -COPY package.json yarn.lock ./ -RUN yarn --ignore-optional --network-timeout 600000 +FROM build as production_buildstage -ARG NODE_ENV=production +COPY package.json yarn.lock ./ +COPY excalidraw-app/package.json ./excalidraw-app/ +COPY packages/excalidraw/package.json ./packages/excalidraw/ + +RUN yarn --network-timeout 600000 COPY . . + +ARG NODE_ENV=production RUN yarn build:app:docker -FROM nginx:1.21-alpine +FROM nginx:1.21-alpine as production -COPY --from=build /opt/node_app/build /usr/share/nginx/html +COPY --from=production_buildstage /opt/node_app/excalidraw-app/build /usr/share/nginx/html HEALTHCHECK CMD wget -q -O /dev/null http://localhost || exit 1 + +FROM build as development \ No newline at end of file diff --git a/README.md b/README.md index e8cd3b06..f2bff40e 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,55 @@ - - - - Excalidraw - - +# Excalidraw without firebase -

- Excalidraw Editor | - Blog | - Documentation | - Excalidraw+ -

+This is a fork from the [excalidraw project](https://github.com/excalidraw/excalidraw) with changes inspired and partly taken from [Kilian Decaderincourt](https://gitlab.com/kiliandeca/excalidraw-fork) to enable support for rooms without using firebase.
-
-

- An open source virtual hand-drawn style whiteboard.
- Collaborative and end-to-end encrypted.
-
-

+## Setup with docker + +Please copy the .env.development.default or .env.production.default file to .env (or with environment without default at the end) and change it according to your needs, see [react-scripts](https://create-react-app.dev/docs/adding-custom-environment-variables/). + +### Development + +``` +docker-compose up -d +docker-compose exec excalidraw yarn install +docker-compose exec excalidraw yarn start +``` + +Hint: Collab mode requires a secure context (https). Localhost works as well, but not http over local network. + +#### Commands + +| Command | Description | +| ------------------ | --------------------------------- | +| `yarn` | Install the dependencies | +| `yarn start` | Run the project | +| `yarn fix` | Reformat all files with Prettier | +| `yarn test` | Run tests | +| `yarn test:update` | Update test snapshots | +| `yarn test:code` | Test for formatting with Prettier | + +### Production + +``` +docker-compose -f docker-compose-prod.yml up -d +``` + + +## Additional licence + +The excalidraw [logo](https://thenounproject.com/icon/2357486/) in this repo – created by [Verry](https://thenounproject.com/verry.dsign.creative) – is licenced under [CC BY 3.0 Unported](https://creativecommons.org/licenses/by/3.0/). +
+ + Excalidraw logo: Sketch handrawn like diagrams. + +

Virtual whiteboard for sketching hand-drawn like diagrams.
Collaborative and end-to-end encrypted.

+

+ + Follow Excalidraw on Twitter + + + Chat with us on Discord + +


@@ -127,3 +159,14 @@ If you like the project, you can become a sponsor at [Open Collective](https://o Last but not least, we're thankful to these companies for offering their services for free: [![Vercel](./.github/assets/vercel.svg)](https://vercel.com) [![Sentry](./.github/assets/sentry.svg)](https://sentry.io) [![Crowdin](./.github/assets/crowdin.svg)](https://crowdin.com) + +## Developers + +You can integrate Excalidraw into your app by installing our [npm component](https://npmjs.com/package/@excalidraw/excalidraw). + +Visit our documentation on [https://docs.excalidraw.com](https://docs.excalidraw.com). + +## Who's integrating Excalidraw + +[Google Cloud](https://googlecloudcheatsheet.withgoogle.com/architecture) • [Meta](https://meta.com/) • [CodeSandbox](https://codesandbox.io/) • [Obsidian Excalidraw](https://github.com/zsviczian/obsidian-excalidraw-plugin) • [Replit](https://replit.com/) • [Slite](https://slite.com/) • [Notion](https://notion.so/) • [HackerRank](https://www.hackerrank.com/) +``` \ No newline at end of file diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml new file mode 100644 index 00000000..982d177b --- /dev/null +++ b/docker-compose-prod.yml @@ -0,0 +1,44 @@ +version: "3.8" + +services: + excalidraw: + stdin_open: true + build: + context: . + target: production + container_name: excalidraw + ports: + - "5010:80" + restart: always + healthcheck: + disable: true + environment: + - NODE_ENV=production + + excalidraw-storage-backend: + build: + context: https://github.com/kitsteam/excalidraw-storage-backend.git#main + target: production + ports: + - "5011:8080" + restart: always + environment: + STORAGE_URI: redis://:${REDIS_PASSWORD}@redis:6379 + STORAGE_TTL: 2592000000 + + excalidraw-room: + image: excalidraw/excalidraw-room + restart: always + ports: + - "5012:80" + + redis: + image: redis + command: redis-server --requirepass ${REDIS_PASSWORD} + restart: always + volumes: + - redis_data:/data + +volumes: + notused: + redis_data: diff --git a/docker-compose.yml b/docker-compose.yml index b82053e5..6c5ecdf6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,24 +2,43 @@ version: "3.8" services: excalidraw: + stdin_open: true build: context: . - args: - - NODE_ENV=development + target: development container_name: excalidraw ports: - - "3000:80" + - "3000:3000" restart: on-failure - stdin_open: true healthcheck: disable: true environment: - NODE_ENV=development volumes: - - ./:/opt/node_app/app:delegated - - ./package.json:/opt/node_app/package.json - - ./yarn.lock:/opt/node_app/yarn.lock - - notused:/opt/node_app/app/node_modules + - node_modules:/opt/node_app/node_modules + - ./:/opt/node_app/ + excalidraw-storage-backend: + stdin_open: true + build: + context: https://github.com/kitsteam/excalidraw-storage-backend.git#main + target: production + ports: + - "8080:8080" + environment: + STORAGE_URI: redis://:${REDIS_PASSWORD}@redis:6379 + STORAGE_TTL: 2592000000 + + excalidraw-room: + image: excalidraw/excalidraw-room + ports: + - "5001:80" + + redis: + image: redis + command: redis-server --requirepass ${REDIS_PASSWORD} + volumes: + - redis_data:/data volumes: - notused: + redis_data: + node_modules: diff --git a/excalidraw-app/App.tsx b/excalidraw-app/App.tsx index 4a3d4284..bed96007 100644 --- a/excalidraw-app/App.tsx +++ b/excalidraw-app/App.tsx @@ -97,6 +97,7 @@ import { AppFooter } from "./components/AppFooter"; import { atom, Provider, useAtom, useAtomValue } from "jotai"; import { useAtomWithInitialValue } from "../packages/excalidraw/jotai"; import { appJotaiStore } from "./app-jotai"; +import { getStorageBackend } from "./data/config"; import "./index.scss"; import { ResolutionType } from "../packages/excalidraw/utility-types"; @@ -354,11 +355,15 @@ const ExcalidrawWrapper = () => { }, [] as FileId[]) || []; if (data.isExternalScene) { - loadFilesFromFirebase( - `${FIREBASE_STORAGE_PREFIXES.shareLinkFiles}/${data.id}`, - data.key, - fileIds, - ).then(({ loadedFiles, erroredFiles }) => { + getStorageBackend() + .then((storageBackend) => { + return storageBackend.loadFilesFromStorageBackend( + `${FIREBASE_STORAGE_PREFIXES.shareLinkFiles}/${data.id}`, + data.key, + fileIds, + ); + }) + .then(({ loadedFiles, erroredFiles }) => { excalidrawAPI.addFiles(loadedFiles); updateStaleImageStatuses({ excalidrawAPI, diff --git a/excalidraw-app/collab/Collab.tsx b/excalidraw-app/collab/Collab.tsx index 9b26af05..83de1723 100644 --- a/excalidraw-app/collab/Collab.tsx +++ b/excalidraw-app/collab/Collab.tsx @@ -83,6 +83,7 @@ import { atom, useAtom } from "jotai"; import { appJotaiStore } from "../app-jotai"; import { Mutable, ValueOf } from "../../packages/excalidraw/utility-types"; import { getVisibleSceneBounds } from "../../packages/excalidraw/element/bounds"; +import { getStorageBackend } from "../data/config"; export const collabAPIAtom = atom(null); export const collabDialogShownAtom = atom(false); @@ -140,7 +141,12 @@ class Collab extends PureComponent { throw new AbortError(); } - return loadFilesFromFirebase(`files/rooms/${roomId}`, roomKey, fileIds); + const storageBackend = await getStorageBackend(); + return storageBackend.loadFilesFromStorageBackend( + `files/rooms/${roomId}`, + roomKey, + fileIds, + ); }, saveFiles: async ({ addedFiles }) => { const { roomId, roomKey } = this.portal; @@ -148,7 +154,8 @@ class Collab extends PureComponent { throw new AbortError(); } - return saveFilesToFirebase({ + const storageBackend = await getStorageBackend(); + return storageBackend.saveFilesToStorageBackend({ prefix: `${FIREBASE_STORAGE_PREFIXES.collabFiles}/${roomId}`, files: await encodeFilesForUpload({ files: addedFiles, @@ -267,11 +274,8 @@ class Collab extends PureComponent { syncableElements: readonly SyncableExcalidrawElement[], ) => { try { - const savedData = await saveToFirebase( - this.portal, - syncableElements, - this.excalidrawAPI.getAppState(), - ); + const storageBackend = await getStorageBackend(); + const savedData = await storageBackend.saveToStorageBackend(this.portal, syncableElements, this.excalidrawAPI.getAppState()); if (this.isCollaborating() && savedData && savedData.reconciledElements) { this.handleRemoteSceneUpdate( @@ -656,11 +660,12 @@ class Collab extends PureComponent { this.excalidrawAPI.resetScene(); try { - const elements = await loadFromFirebase( - roomLinkData.roomId, - roomLinkData.roomKey, - this.portal.socket, - ); + const storageBackend = await getStorageBackend(); + const elements = await storageBackend.loadFromStorageBackend( + roomLinkData.roomId, + roomLinkData.roomKey, + this.portal.socket, + ); if (elements) { this.setLastBroadcastedOrReceivedSceneVersion( getSceneVersion(elements), diff --git a/excalidraw-app/collab/Portal.tsx b/excalidraw-app/collab/Portal.tsx index bf8ffa5d..6570944c 100644 --- a/excalidraw-app/collab/Portal.tsx +++ b/excalidraw-app/collab/Portal.tsx @@ -23,7 +23,7 @@ import type { Socket } from "socket.io-client"; class Portal { collab: TCollabClass; - socket: Socket | null = null; + socket: SocketIOClient.Socket | null = null; socketInitialized: boolean = false; // we don't want the socket to emit any updates until it is fully initialized roomId: string | null = null; roomKey: string | null = null; @@ -33,7 +33,7 @@ class Portal { this.collab = collab; } - open(socket: Socket, id: string, key: string) { + open(socket: SocketIOClient.Socket, id: string, key: string) { this.socket = socket; this.roomId = id; this.roomKey = key; diff --git a/excalidraw-app/data/StorageBackend.ts b/excalidraw-app/data/StorageBackend.ts new file mode 100644 index 00000000..7aeff9c1 --- /dev/null +++ b/excalidraw-app/data/StorageBackend.ts @@ -0,0 +1,45 @@ +import { SyncableExcalidrawElement } from "."; +import { ExcalidrawElement, FileId } from "../../packages/excalidraw/element/types"; +import { AppState, BinaryFileData } from "../../packages/excalidraw/types"; +import Portal from "../collab/Portal"; + +export interface StorageBackend { + isSaved: (portal: Portal, elements: readonly ExcalidrawElement[]) => boolean; + saveToStorageBackend: ( + portal: Portal, + elements: readonly SyncableExcalidrawElement[], + appState: AppState, + ) => Promise; + loadFromStorageBackend: ( + roomId: string, + roomKey: string, + socket: SocketIOClient.Socket | null, + ) => Promise; + saveFilesToStorageBackend: ({ + prefix, + files, + }: { + prefix: string; + files: { + id: FileId; + buffer: Uint8Array; + }[]; + }) => Promise<{ + savedFiles: Map; + erroredFiles: Map; + }>; + loadFilesFromStorageBackend: ( + prefix: string, + decryptionKey: string, + filesIds: readonly FileId[], + ) => Promise<{ + loadedFiles: BinaryFileData[]; + erroredFiles: Map; + }>; +} + +export interface StoredScene { + sceneVersion: number; + iv: Uint8Array; + ciphertext: ArrayBuffer; +} diff --git a/excalidraw-app/data/config.ts b/excalidraw-app/data/config.ts new file mode 100644 index 00000000..7391c134 --- /dev/null +++ b/excalidraw-app/data/config.ts @@ -0,0 +1,54 @@ +import { + isSavedToFirebase, + loadFilesFromFirebase, + loadFromFirebase, + saveFilesToFirebase, + saveToFirebase, + } from "./firebase"; + import { + isSavedToHttpStorage, + loadFilesFromHttpStorage, + loadFromHttpStorage, + saveFilesToHttpStorage, + saveToHttpStorage, + } from "./httpStorage"; + import { StorageBackend } from "./StorageBackend"; + + const firebaseStorage: StorageBackend = { + isSaved: isSavedToFirebase, + saveToStorageBackend: saveToFirebase, + loadFromStorageBackend: loadFromFirebase, + saveFilesToStorageBackend: saveFilesToFirebase, + loadFilesFromStorageBackend: loadFilesFromFirebase, + }; + + const httpStorage: StorageBackend = { + isSaved: isSavedToHttpStorage, + saveToStorageBackend: saveToHttpStorage, + loadFromStorageBackend: loadFromHttpStorage, + saveFilesToStorageBackend: saveFilesToHttpStorage, + loadFilesFromStorageBackend: loadFilesFromHttpStorage, + }; + + const storageBackends = new Map() + .set("firebase", firebaseStorage) + .set("http", httpStorage); + + export let storageBackend: StorageBackend | null = null; + + export async function getStorageBackend() { + if (storageBackend) { + return storageBackend; + } + + const storageBackendName = import.meta.env.VITE_APP_STORAGE_BACKEND || ''; + + if (storageBackends.has(storageBackendName)) { + storageBackend = storageBackends.get(storageBackendName) as StorageBackend; + } else { + console.warn("No storage backend found, default to firebase"); + storageBackend = firebaseStorage; + } + + return storageBackend; + } \ No newline at end of file diff --git a/excalidraw-app/data/firebase.ts b/excalidraw-app/data/firebase.ts index f37fbbd8..9351a783 100644 --- a/excalidraw-app/data/firebase.ts +++ b/excalidraw-app/data/firebase.ts @@ -21,7 +21,7 @@ import { MIME_TYPES } from "../../packages/excalidraw/constants"; import { reconcileElements } from "../collab/reconciliation"; import { getSyncableElements, SyncableExcalidrawElement } from "."; import { ResolutionType } from "../../packages/excalidraw/utility-types"; -import type { Socket } from "socket.io-client"; +import { Socket } from "socket.io-client"; // private // ----------------------------------------------------------------------------- @@ -49,8 +49,10 @@ const _loadFirebase = async () => { const firebase = ( await import(/* webpackChunkName: "firebase" */ "firebase/app") ).default; + const storage = import.meta.env.VITE_APP_STORAGE_BACKEND; + const useFirebase = storage === "firebase"; - if (!isFirebaseInitialized) { + if (useFirebase && !isFirebaseInitialized) { try { firebase.initializeApp(FIREBASE_CONFIG); } catch (error: any) { @@ -139,12 +141,12 @@ const decryptElements = async ( }; class FirebaseSceneVersionCache { - private static cache = new WeakMap(); - static get = (socket: Socket) => { + private static cache = new WeakMap(); + static get = (socket: SocketIOClient.Socket) => { return FirebaseSceneVersionCache.cache.get(socket); }; static set = ( - socket: Socket, + socket: SocketIOClient.Socket, elements: readonly SyncableExcalidrawElement[], ) => { FirebaseSceneVersionCache.cache.set(socket, getSceneVersion(elements)); @@ -286,7 +288,7 @@ export const saveToFirebase = async ( export const loadFromFirebase = async ( roomId: string, roomKey: string, - socket: Socket | null, + socket: SocketIOClient.Socket | null, ): Promise => { const firebase = await loadFirestore(); const db = firebase.firestore(); diff --git a/excalidraw-app/data/httpStorage.ts b/excalidraw-app/data/httpStorage.ts new file mode 100644 index 00000000..4a3c89e2 --- /dev/null +++ b/excalidraw-app/data/httpStorage.ts @@ -0,0 +1,266 @@ +// Inspired and partly copied from https://gitlab.com/kiliandeca/excalidraw-fork +// MIT, Kilian Decaderincourt + +import { getSyncableElements, SyncableExcalidrawElement } from "."; +import { MIME_TYPES } from "../../packages/excalidraw/constants"; +import { decompressData } from "../../packages/excalidraw/data/encode"; +import { encryptData, decryptData, IV_LENGTH_BYTES } from "../../packages/excalidraw/data/encryption"; +import { restoreElements } from "../../packages/excalidraw/data/restore"; +import { getSceneVersion } from "../../packages/excalidraw/element"; +import { ExcalidrawElement, FileId } from "../../packages/excalidraw/element/types"; +import { + AppState, + BinaryFileData, + BinaryFileMetadata, + DataURL, +} from "../../packages/excalidraw/types"; +import Portal from "../collab/Portal"; +import { reconcileElements } from "../collab/reconciliation"; +import { StoredScene } from "./StorageBackend"; + +const HTTP_STORAGE_BACKEND_URL = import.meta.env.VITE_APP_HTTP_STORAGE_BACKEND_URL; +const SCENE_VERSION_LENGTH_BYTES = 4 + +// There is a lot of intentional duplication with the firebase file +// to prevent modifying upstream files and ease futur maintenance of this fork + +const httpStorageSceneVersionCache = new WeakMap< + SocketIOClient.Socket, + number +>(); + +export const isSavedToHttpStorage = ( + portal: Portal, + elements: readonly ExcalidrawElement[], +): boolean => { + if (portal.socket && portal.roomId && portal.roomKey) { + const sceneVersion = getSceneVersion(elements); + + return httpStorageSceneVersionCache.get(portal.socket) === sceneVersion; + } + // if no room exists, consider the room saved so that we don't unnecessarily + // prevent unload (there's nothing we could do at that point anyway) + return true; +}; + +export const saveToHttpStorage = async ( + portal: Portal, + elements: readonly SyncableExcalidrawElement[], + appState: AppState, +) => { + const { roomId, roomKey, socket } = portal; + if ( + // if no room exists, consider the room saved because there's nothing we can + // do at this point + !roomId || + !roomKey || + !socket || + isSavedToHttpStorage(portal, elements) + ) { + return false; + } + + const sceneVersion = getSceneVersion(elements); + const getResponse = await fetch( + `${HTTP_STORAGE_BACKEND_URL}/rooms/${roomId}`, + ); + + if (!getResponse.ok && getResponse.status !== 404) { + return false; + }; + + if(getResponse.status === 404) { + const result: boolean = await saveElementsToBackend(roomKey, roomId, [...elements], sceneVersion) + if(result) { + return { + reconciledElements: null + } + } + return false + }; + + // If room already exist, we compare scene versions to check + // if we're up to date before saving our scene + const buffer = await getResponse.arrayBuffer(); + const sceneVersionFromRequest = parseSceneVersionFromRequest(buffer); + if (sceneVersionFromRequest >= sceneVersion) { + return false; + } + + const existingElements = await getElementsFromBuffer(buffer, roomKey); + const reconciledElements = getSyncableElements( + reconcileElements(elements, existingElements, appState), + ); + + const result: boolean = await saveElementsToBackend(roomKey, roomId, reconciledElements, sceneVersion) + + if (result) { + httpStorageSceneVersionCache.set(socket, sceneVersion); + return { + reconciledElements: elements + }; + } + return false; +}; + +export const loadFromHttpStorage = async ( + roomId: string, + roomKey: string, + socket: SocketIOClient.Socket | null, +): Promise => { + const HTTP_STORAGE_BACKEND_URL = import.meta.env.VITE_APP_HTTP_STORAGE_BACKEND_URL; + const getResponse = await fetch( + `${HTTP_STORAGE_BACKEND_URL}/rooms/${roomId}`, + ); + + const buffer = await getResponse.arrayBuffer(); + const elements = await getElementsFromBuffer(buffer, roomKey); + + if (socket) { + httpStorageSceneVersionCache.set(socket, getSceneVersion(elements)); + } + + return restoreElements(elements, null); +}; + +const getElementsFromBuffer = async ( + buffer: ArrayBuffer, + key: string, +): Promise => { + // Buffer should contain both the IV (fixed length) and encrypted data + const sceneVersion = parseSceneVersionFromRequest(buffer); + const iv = new Uint8Array(buffer.slice(SCENE_VERSION_LENGTH_BYTES, IV_LENGTH_BYTES + SCENE_VERSION_LENGTH_BYTES)); + const encrypted = buffer.slice(IV_LENGTH_BYTES + SCENE_VERSION_LENGTH_BYTES, buffer.byteLength); + + return await decryptElements( + { sceneVersion: sceneVersion, ciphertext: encrypted, iv }, + key + ); +}; + +export const saveFilesToHttpStorage = async ({ + prefix, + files, +}: { + prefix: string; + files: { id: FileId; buffer: Uint8Array }[]; +}) => { + const erroredFiles = new Map(); + const savedFiles = new Map(); + + const HTTP_STORAGE_BACKEND_URL = import.meta.env.VITE_APP_HTTP_STORAGE_BACKEND_URL; + + await Promise.all( + files.map(async ({ id, buffer }) => { + try { + const payloadBlob = new Blob([buffer]); + const payload = await new Response(payloadBlob).arrayBuffer(); + await fetch(`${HTTP_STORAGE_BACKEND_URL}/files/${id}`, { + method: "PUT", + body: payload, + }); + savedFiles.set(id, true); + } catch (error: any) { + erroredFiles.set(id, true); + } + }), + ); + + return { savedFiles, erroredFiles }; +}; + +export const loadFilesFromHttpStorage = async ( + prefix: string, + decryptionKey: string, + filesIds: readonly FileId[], +) => { + const loadedFiles: BinaryFileData[] = []; + const erroredFiles = new Map(); + + ////////////// + await Promise.all( + [...new Set(filesIds)].map(async (id) => { + try { + const HTTP_STORAGE_BACKEND_URL = import.meta.env.VITE_APP_HTTP_STORAGE_BACKEND_URL; + const response = await fetch(`${HTTP_STORAGE_BACKEND_URL}/files/${id}`); + if (response.status < 400) { + const arrayBuffer = await response.arrayBuffer(); + + const { data, metadata } = await decompressData( + new Uint8Array(arrayBuffer), + { + decryptionKey, + }, + ); + + const dataURL = new TextDecoder().decode(data) as DataURL; + + loadedFiles.push({ + mimeType: metadata.mimeType || MIME_TYPES.binary, + id, + dataURL, + created: metadata?.created || Date.now(), + }); + } else { + erroredFiles.set(id, true); + } + } catch (error: any) { + erroredFiles.set(id, true); + console.error(error); + } + }), + ); + ////// + + return { loadedFiles, erroredFiles }; +}; + +const saveElementsToBackend = async (roomKey: string, roomId: string, elements: SyncableExcalidrawElement[], sceneVersion: number) => { + const { ciphertext, iv } = await encryptElements(roomKey, elements); + + // Concatenate Scene Version, IV with encrypted data (IV does not have to be secret). + const numberBuffer = new ArrayBuffer(4); + const numberView = new DataView(numberBuffer); + numberView.setUint32(0, sceneVersion, false); + const sceneVersionBuffer = numberView.buffer; + const payloadBlob = await new Response(new Blob([sceneVersionBuffer, iv.buffer, ciphertext])).arrayBuffer(); + const putResponse = await fetch( + `${HTTP_STORAGE_BACKEND_URL}/rooms/${roomId}`, + { + method: "PUT", + body: payloadBlob, + }, + ); + + return putResponse.ok +} + +const parseSceneVersionFromRequest = (buffer: ArrayBuffer) => { + const view = new DataView(buffer); + return view.getUint32(0, false); +} + +const decryptElements = async ( + data: StoredScene, + roomKey: string, +): Promise => { + const ciphertext = data.ciphertext; + const iv = data.iv; + + const decrypted = await decryptData(iv, ciphertext, roomKey); + const decodedData = new TextDecoder("utf-8").decode( + new Uint8Array(decrypted), + ); + return JSON.parse(decodedData); +}; + +const encryptElements = async ( + key: string, + elements: readonly ExcalidrawElement[], +): Promise<{ ciphertext: ArrayBuffer; iv: Uint8Array }> => { + const json = JSON.stringify(elements); + const encoded = new TextEncoder().encode(json); + const { encryptedBuffer, iv } = await encryptData(key, encoded); + + return { ciphertext: encryptedBuffer, iv }; +}; \ No newline at end of file diff --git a/excalidraw-app/data/index.ts b/excalidraw-app/data/index.ts index 0f54ee88..4074d04e 100644 --- a/excalidraw-app/data/index.ts +++ b/excalidraw-app/data/index.ts @@ -4,6 +4,7 @@ import { } from "../../packages/excalidraw/data/encode"; import { decryptData, + encryptData, generateEncryptionKey, IV_LENGTH_BYTES, } from "../../packages/excalidraw/data/encryption"; @@ -33,7 +34,7 @@ import { WS_SUBTYPES, } from "../app_constants"; import { encodeFilesForUpload } from "./FileManager"; -import { saveFilesToFirebase } from "./firebase"; +import { getStorageBackend } from "./config"; export type SyncableExcalidrawElement = ExcalidrawElement & { _brand: "SyncableExcalidrawElement"; @@ -343,7 +344,8 @@ export const exportToBackend = async ( url.hash = `json=${json.id},${encryptionKey}`; const urlString = url.toString(); - await saveFilesToFirebase({ + const storageBackend = await getStorageBackend(); + await storageBackend.saveFilesToStorageBackend({ prefix: `/files/shareLinks/${json.id}`, files: filesToUpload, }); diff --git a/excalidraw-app/package.json b/excalidraw-app/package.json index cc133581..5e8df87a 100644 --- a/excalidraw-app/package.json +++ b/excalidraw-app/package.json @@ -30,7 +30,7 @@ "private": true, "scripts": { "build-node": "node ./scripts/build-node.js", - "build:app:docker": "cross-env VITE_APP_DISABLE_SENTRY=true VITE_APP_DISABLE_TRACKING=true vite build", + "build:app:docker": "cross-env VITE_APP_DISABLE_SENTRY=true VITE_APP_DISABLE_TRACKING=true VITE_APP_ENABLE_ESLINT=false vite build", "build:app": "cross-env VITE_APP_GIT_SHA=$VERCEL_GIT_COMMIT_SHA vite build", "build:version": "node ../scripts/build-version.js", "build": "yarn build:app && yarn build:version", diff --git a/excalidraw-app/vite-env.d.ts b/excalidraw-app/vite-env.d.ts index 3230946f..a9950e3e 100644 --- a/excalidraw-app/vite-env.d.ts +++ b/excalidraw-app/vite-env.d.ts @@ -35,6 +35,9 @@ interface ImportMetaEnv { VITE_APP_GIT_SHA: string; + VITE_APP_HTTP_STORAGE_BACKEND_URL: string; + VITE_APP_STORAGE_BACKEND: "http" | "firebase"; + MODE: string; DEV: string; diff --git a/excalidraw-app/vite.config.mts b/excalidraw-app/vite.config.mts index b3714e81..7d1e5711 100644 --- a/excalidraw-app/vite.config.mts +++ b/excalidraw-app/vite.config.mts @@ -10,9 +10,8 @@ const envVars = loadEnv("", `../`); // https://vitejs.dev/config/ export default defineConfig({ server: { - port: Number(envVars.VITE_APP_PORT || 3000), - // open the browser - open: true, + host: '0.0.0.0', + port: Number(envVars.VITE_APP_PORT || 3000) }, // We need to specify the envDir since now there are no //more located in parallel with the vite.config.ts file but in parent dir diff --git a/package.json b/package.json index 84be8c55..834e4358 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "react": "18.2.0", "react-dom": "18.2.0", "socket.io-client": "4.7.2" + }, "devDependencies": { "@excalidraw/eslint-config": "1.0.3", @@ -58,9 +59,9 @@ "prettier": "@excalidraw/prettier-config", "scripts": { "build-node": "node ./scripts/build-node.js", - "build:app:docker": "cross-env VITE_APP_DISABLE_SENTRY=true VITE_APP_DISABLE_TRACKING=true vite build", - "build:app": "cross-env VITE_APP_GIT_SHA=$VERCEL_GIT_COMMIT_SHA vite build", - "build:version": "node ./scripts/build-version.js", + "build:app:docker": "yarn --cwd ./excalidraw-app build:app:docker", + "build:app": "yarn --cwd ./excalidraw-app build:app", + "build:version": "yarn --cwd ./excalidraw-app build:version", "build": "yarn --cwd ./excalidraw-app build", "fix:code": "yarn test:code --fix", "fix:other": "yarn prettier --write",