From 0e5c29b3f31de90344117479954ee19327d31426 Mon Sep 17 00:00:00 2001 From: Edwin Lin Date: Mon, 9 Mar 2020 08:48:25 -0700 Subject: [PATCH] basic Socket.io implementation of collaborative editing (#879) * Enable collaborative syncing for elements * Don't fall back to local storage if using a room, as that is confusing * Use remote socket server * Send updates to new users when they join * ~ * add mouse tracking * enable collaboration, rooms, and mouse tracking * fix syncing bugs and add a button to start syncing mid session * enable collaboration, rooms, and mouse tracking * fix syncing bugs and add a button to start syncing mid session * Add Live button and app state to support tracking collaborator counts * Enable collaborative syncing for elements * add mouse tracking * enable collaboration, rooms, and mouse tracking * fix syncing bugs and add a button to start syncing mid session * fix syncing bugs and add a button to start syncing mid session * Add Live button and app state to support tracking collaborator counts * prettier * Fix bug with remote pointers not changing on scroll * Enable collaborative syncing for elements * add mouse tracking * enable collaboration, rooms, and mouse tracking * fix syncing bugs and add a button to start syncing mid session * enable collaboration, rooms, and mouse tracking * fix syncing bugs and add a button to start syncing mid session * Add Live button and app state to support tracking collaborator counts * enable collaboration, rooms, and mouse tracking * fix syncing bugs and add a button to start syncing mid session * fix syncing bugs and add a button to start syncing mid session * Fix bug with remote pointers not changing on scroll * remove UI for collaboration * remove link * clean up lingering unused UI * set random IV passed per encrypted message, reduce room id length, refactored socket broadcasting API, rename room_id to room, removed throttling of pointer movement * fix package.json conflict --- package-lock.json | 238 ++++++++++++++++++++++++++++++++++++ package.json | 2 + src/appState.ts | 4 + src/components/App.tsx | 184 ++++++++++++++++++++++++++-- src/components/icons.tsx | 6 + src/data/index.ts | 151 ++++++++++++++++++++--- src/data/localStorage.ts | 3 + src/locales/en.json | 1 + src/renderer/renderScene.ts | 9 ++ src/scene/export.ts | 1 + src/scene/types.ts | 1 + src/types.ts | 3 + 12 files changed, 575 insertions(+), 28 deletions(-) diff --git a/package-lock.json b/package-lock.json index 59ac0b5d..0bbbaa8e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1936,6 +1936,21 @@ "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.4.tgz", "integrity": "sha512-8+KAKzEvSUdeo+kmqnKrqgeE+LcA0tjYWFY7RPProVYwnqDjukzO+3b6dLD56rYX5TdWejnEOLJYOIeh4CXKuA==" }, + "@types/lodash": { + "version": "4.14.149", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.149.tgz", + "integrity": "sha512-ijGqzZt/b7BfzcK9vTrS6MFljQRPn5BFWOx8oE0GYxribu6uV+aA9zZuXI1zc/etK9E8nrgdoF2+LgUw7+9tJQ==", + "dev": true + }, + "@types/lodash.throttle": { + "version": "4.1.6", + "resolved": "https://registry.npmjs.org/@types/lodash.throttle/-/lodash.throttle-4.1.6.tgz", + "integrity": "sha512-/UIH96i/sIRYGC60NoY72jGkCJtFN5KVPhEMMMTjol65effe1gPn0tycJqV5tlSwMTzX8FqzB5yAj0rfGHTPNg==", + "dev": true, + "requires": { + "@types/lodash": "*" + } + }, "@types/minimatch": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", @@ -1990,6 +2005,12 @@ "@types/react": "*" } }, + "@types/socket.io-client": { + "version": "1.4.32", + "resolved": "https://registry.npmjs.org/@types/socket.io-client/-/socket.io-client-1.4.32.tgz", + "integrity": "sha512-Vs55Kq8F+OWvy1RLA31rT+cAyemzgm0EWNeax6BWF8H7QiiOYMJIdcwSDdm5LVgfEkoepsWkS+40+WNb7BUMbg==", + "dev": true + }, "@types/stack-utils": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-1.0.1.tgz", @@ -2412,6 +2433,11 @@ } } }, + "after": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/after/-/after-0.8.2.tgz", + "integrity": "sha1-/ts5T58OAqqXaOcCvaI7UF+ufh8=" + }, "aggregate-error": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.0.1.tgz", @@ -2630,6 +2656,11 @@ "es-abstract": "^1.17.0-next.1" } }, + "arraybuffer.slice": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz", + "integrity": "sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog==" + }, "arrify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", @@ -3040,6 +3071,11 @@ "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.18.0.tgz", "integrity": "sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ==" }, + "backo2": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz", + "integrity": "sha1-MasayLEpNjRj41s+u2n038+6eUc=" + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -3100,6 +3136,11 @@ } } }, + "base64-arraybuffer": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-0.1.5.tgz", + "integrity": "sha1-c5JncZI7Whl0etZmqlzUv5xunOg=" + }, "base64-js": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", @@ -3118,6 +3159,14 @@ "tweetnacl": "^0.14.3" } }, + "better-assert": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/better-assert/-/better-assert-1.0.2.tgz", + "integrity": "sha1-QIZrnhueC1W0gYlDEeaPr/rrxSI=", + "requires": { + "callsite": "1.0.0" + } + }, "big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -3137,6 +3186,11 @@ "file-uri-to-path": "1.0.0" } }, + "blob": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/blob/-/blob-0.0.5.tgz", + "integrity": "sha512-gaqbzQPqOoamawKg0LGVd7SzLgXS+JH61oWprSLH+P+abTczqJbhTR8CmJ2u9/bUYNmHTGJx/UEmn6doAvvuig==" + }, "block-stream": { "version": "0.0.9", "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.9.tgz", @@ -3472,6 +3526,11 @@ "caller-callsite": "^2.0.0" } }, + "callsite": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz", + "integrity": "sha1-KAOY5dZkvXQDi28JBRU+borxvCA=" + }, "callsites": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", @@ -3897,11 +3956,21 @@ "integrity": "sha512-W6Af2Iw1z4CB7q4uU4hv646dW9GQuBM+YpC0UvUCWSD8w90SJjp+ujJuXaEMtAXBtSqGfMPuFOVn4/+FlaqfBA==", "dev": true }, + "component-bind": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/component-bind/-/component-bind-1.0.0.tgz", + "integrity": "sha1-AMYIq33Nk4l8AAllGx06jh5zu9E=" + }, "component-emitter": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==" }, + "component-inherit": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/component-inherit/-/component-inherit-0.0.3.tgz", + "integrity": "sha1-ZF/ErfWLcrZJ1crmUTVhnbJv8UM=" + }, "compose-function": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/compose-function/-/compose-function-3.0.3.tgz", @@ -4969,6 +5038,51 @@ "once": "^1.4.0" } }, + "engine.io-client": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.4.0.tgz", + "integrity": "sha512-a4J5QO2k99CM2a0b12IznnyQndoEvtA4UAldhGzKqnHf42I3Qs2W5SPnDvatZRcMaNZs4IevVicBPayxYt6FwA==", + "requires": { + "component-emitter": "1.2.1", + "component-inherit": "0.0.3", + "debug": "~4.1.0", + "engine.io-parser": "~2.2.0", + "has-cors": "1.1.0", + "indexof": "0.0.1", + "parseqs": "0.0.5", + "parseuri": "0.0.5", + "ws": "~6.1.0", + "xmlhttprequest-ssl": "~1.5.4", + "yeast": "0.1.2" + }, + "dependencies": { + "component-emitter": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", + "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=" + }, + "ws": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz", + "integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==", + "requires": { + "async-limiter": "~1.0.0" + } + } + } + }, + "engine.io-parser": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.2.0.tgz", + "integrity": "sha512-6I3qD9iUxotsC5HEMuuGsKA0cXerGz+4uGcXQEkfBidgKf0amsjrrtwcbwK/nzpZBxclXlV7gGl9dgWvu4LF6w==", + "requires": { + "after": "0.8.2", + "arraybuffer.slice": "~0.0.7", + "base64-arraybuffer": "0.1.5", + "blob": "0.0.5", + "has-binary2": "~1.0.2" + } + }, "enhanced-resolve": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.1.1.tgz", @@ -6625,6 +6739,26 @@ } } }, + "has-binary2": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-binary2/-/has-binary2-1.0.3.tgz", + "integrity": "sha512-G1LWKhDSvhGeAQ8mPVQlqNcOB2sJdwATtZKl2pDKKHfpf/rYj24lkinxf69blJbnsvtqqNU+L3SL50vzZhXOnw==", + "requires": { + "isarray": "2.0.1" + }, + "dependencies": { + "isarray": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.1.tgz", + "integrity": "sha1-o32U7ZzaLVmGXJ92/llu4fM4dB4=" + } + } + }, + "has-cors": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-cors/-/has-cors-1.1.0.tgz", + "integrity": "sha1-XkdHk/fqmEPRu5nCPu9J/xJv/zk=" + }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -7190,6 +7324,11 @@ "resolved": "https://registry.npmjs.org/indexes-of/-/indexes-of-1.0.1.tgz", "integrity": "sha1-8w9xbI4r00bHtn0985FVZqfAVgc=" }, + "indexof": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/indexof/-/indexof-0.0.1.tgz", + "integrity": "sha1-gtwzbSMrkGIXnQWrMpOmYFn9Q10=" + }, "infer-owner": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz", @@ -10503,6 +10642,11 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" }, + "object-component": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/object-component/-/object-component-0.0.3.tgz", + "integrity": "sha1-8MaapQ78lbhmwYb0AKM3acsvEpE=" + }, "object-copy": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", @@ -10918,6 +11062,22 @@ "resolved": "https://registry.npmjs.org/parse5/-/parse5-4.0.0.tgz", "integrity": "sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA==" }, + "parseqs": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.5.tgz", + "integrity": "sha1-1SCKNzjkZ2bikbouoXNoSSGouJ0=", + "requires": { + "better-assert": "~1.0.0" + } + }, + "parseuri": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.5.tgz", + "integrity": "sha1-gCBKUNTbt3m/3G6+J3jZDkvOMgo=", + "requires": { + "better-assert": "~1.0.0" + } + }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -14219,6 +14379,69 @@ "kind-of": "^3.2.0" } }, + "socket.io-client": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-2.3.0.tgz", + "integrity": "sha512-cEQQf24gET3rfhxZ2jJ5xzAOo/xhZwK+mOqtGRg5IowZsMgwvHwnf/mCRapAAkadhM26y+iydgwsXGObBB5ZdA==", + "requires": { + "backo2": "1.0.2", + "base64-arraybuffer": "0.1.5", + "component-bind": "1.0.0", + "component-emitter": "1.2.1", + "debug": "~4.1.0", + "engine.io-client": "~3.4.0", + "has-binary2": "~1.0.2", + "has-cors": "1.1.0", + "indexof": "0.0.1", + "object-component": "0.0.3", + "parseqs": "0.0.5", + "parseuri": "0.0.5", + "socket.io-parser": "~3.3.0", + "to-array": "0.1.4" + }, + "dependencies": { + "component-emitter": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", + "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=" + } + } + }, + "socket.io-parser": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-3.3.0.tgz", + "integrity": "sha512-hczmV6bDgdaEbVqhAeVMM/jfUfzuEZHsQg6eOmLgJht6G3mPKMxYm75w2+qhAQZ+4X+1+ATZ+QFKeOZD5riHng==", + "requires": { + "component-emitter": "1.2.1", + "debug": "~3.1.0", + "isarray": "2.0.1" + }, + "dependencies": { + "component-emitter": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", + "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=" + }, + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "requires": { + "ms": "2.0.0" + } + }, + "isarray": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.1.tgz", + "integrity": "sha1-o32U7ZzaLVmGXJ92/llu4fM4dB4=" + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } + } + }, "sockjs": { "version": "0.3.19", "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.19.tgz", @@ -15114,6 +15337,11 @@ "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.4.tgz", "integrity": "sha1-I2QN17QtAEM5ERQIIOXPRA5SHdE=" }, + "to-array": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/to-array/-/to-array-0.1.4.tgz", + "integrity": "sha1-F+bBH3PdTz10zaek/zI46a2b+JA=" + }, "to-arraybuffer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz", @@ -17469,6 +17697,11 @@ "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" }, + "xmlhttprequest-ssl": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz", + "integrity": "sha1-wodrBhaKrcQOV9l+gRkayPQ5iz4=" + }, "xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", @@ -17539,6 +17772,11 @@ "camelcase": "^5.0.0", "decamelize": "^1.2.0" } + }, + "yeast": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/yeast/-/yeast-0.1.2.tgz", + "integrity": "sha1-AI4G2AlDIMNy28L47XagymyKxBk=" } } } diff --git a/package.json b/package.json index e034ccc4..62030a74 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "react-dom": "16.13.0", "react-scripts": "3.4.0", "roughjs": "4.0.4", + "socket.io-client": "2.3.0", "stacktrace-js": "2.0.2" }, "devDependencies": { @@ -22,6 +23,7 @@ "@types/nanoid": "2.1.0", "@types/react": "16.9.23", "@types/react-dom": "16.9.5", + "@types/socket.io-client": "1.4.32", "asar": "2.1.0", "eslint": "6.8.0", "eslint-config-prettier": "6.10.0", diff --git a/src/appState.ts b/src/appState.ts index fd88f99a..c159778f 100644 --- a/src/appState.ts +++ b/src/appState.ts @@ -1,5 +1,6 @@ import { AppState, FlooredNumber } from "./types"; import { getDateTime } from "./utils"; +import { getCollaborationLinkData } from "./data"; const DEFAULT_PROJECT_NAME = `excalidraw-${getDateTime()}`; export const DEFAULT_FONT = "20px Virgil"; @@ -27,12 +28,15 @@ export function getDefaultAppState(): AppState { cursorY: 0, scrolledOutside: false, name: DEFAULT_PROJECT_NAME, + isCollaborating: !!getCollaborationLinkData(window.location.href), isResizing: false, selectionElement: null, zoom: 1, openMenu: null, lastPointerDownWith: "mouse", selectedElementIds: {}, + remotePointers: {}, + collaboratorCount: 0, }; } diff --git a/src/components/App.tsx b/src/components/App.tsx index e3b63b6d..ffa5997c 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -1,5 +1,6 @@ import React from "react"; +import socketIOClient from "socket.io-client"; import rough from "roughjs/bin/rough"; import { RoughCanvas } from "roughjs/bin/canvas"; import { Point } from "roughjs/bin/geometry"; @@ -29,7 +30,16 @@ import { getSelectedElements, isSomeElementSelected, } from "../scene"; -import { saveToLocalStorage, loadScene, loadFromBlob } from "../data"; +import { + decryptAESGEM, + encryptAESGEM, + saveToLocalStorage, + loadScene, + loadFromBlob, + SOCKET_SERVER, + SocketUpdateData, +} from "../data"; +import { restore } from "../data/restore"; import { renderScene } from "../renderer"; import { AppState, GestureEvent, Gesture } from "../types"; @@ -77,6 +87,7 @@ import { import { LayerUI } from "./LayerUI"; import { ScrollBars } from "../scene/types"; import { invalidateShapeForElement } from "../renderer/renderElement"; +import { generateCollaborationLink, getCollaborationLinkData } from "../data"; // ----------------------------------------------------------------------------- // TEST HOOKS @@ -88,12 +99,15 @@ declare global { elements: typeof elements; appState: AppState; }; + // TEMPORARY until we have a UI to support this + generateCollaborationLink: () => Promise; } } if (process.env.NODE_ENV === "test") { window.__TEST__ = {} as Window["__TEST__"]; } +window.generateCollaborationLink = generateCollaborationLink; // ----------------------------------------------------------------------------- @@ -136,6 +150,10 @@ function setCursorForShape(shape: string) { export class App extends React.Component { canvas: HTMLCanvasElement | null = null; rc: RoughCanvas | null = null; + socket: SocketIOClient.Socket | null = null; + socketInitialized: boolean = false; // we don't want the socket to emit any updates until it is fully initalized + roomID: string | null = null; + roomKey: string | null = null; actionManager: ActionManager; canvasOnlyActions = ["selectAll"]; @@ -207,6 +225,96 @@ export class App extends React.Component { event.preventDefault(); }; + private initializeSocketClient = () => { + if (this.socket) { + return; + } + const roomMatch = getCollaborationLinkData(window.location.href); + if (roomMatch) { + this.socket = socketIOClient(SOCKET_SERVER); + this.roomID = roomMatch[1]; + this.roomKey = roomMatch[2]; + this.socket.on("init-room", () => { + this.socket && this.socket.emit("join-room", this.roomID); + }); + this.socket.on( + "client-broadcast", + async (encryptedData: ArrayBuffer, iv: Uint8Array) => { + if (!this.roomKey) { + return; + } + const decryptedData = await decryptAESGEM( + encryptedData, + this.roomKey, + iv, + ); + + switch (decryptedData.type) { + case "INVALID_RESPONSE": + return; + case "SCENE_UPDATE": + const { + elements: sceneElements, + appState: sceneAppState, + } = decryptedData.payload; + const restoredState = restore( + sceneElements || [], + sceneAppState || getDefaultAppState(), + { scrollToContent: true }, + ); + elements = restoredState.elements; + this.setState({}); + if (this.socketInitialized === false) { + this.socketInitialized = true; + } + break; + case "MOUSE_LOCATION": + const { socketID, pointerCoords } = decryptedData.payload; + this.setState({ + remotePointers: { + ...this.state.remotePointers, + [socketID]: pointerCoords, + }, + }); + break; + } + }, + ); + this.socket.on("first-in-room", () => { + if (this.socket) { + this.socket.off("first-in-room"); + } + this.socketInitialized = true; + }); + this.socket.on("room-user-count", (collaboratorCount: number) => { + this.setState({ collaboratorCount }); + }); + this.socket.on("new-user", async (socketID: string) => { + this.broadcastSocketData({ + type: "SCENE_UPDATE", + payload: { + elements, + appState: this.state, + }, + }); + }); + } + }; + + private broadcastSocketData = async (data: SocketUpdateData) => { + if (this.socketInitialized && this.socket && this.roomID && this.roomKey) { + const json = JSON.stringify(data); + const encoded = new TextEncoder().encode(json); + const encrypted = await encryptAESGEM(encoded, this.roomKey); + this.socket.emit( + "server-broadcast", + this.roomID, + encrypted.data, + encrypted.iv, + ); + } + }; + private unmounted = false; public async componentDidMount() { if (process.env.NODE_ENV === "test") { @@ -251,18 +359,24 @@ export class App extends React.Component { // Backwards compatibility with legacy url format const scene = await loadScene(id); this.syncActionResult(scene); - } else { - const match = window.location.hash.match( - /^#json=([0-9]+),([a-zA-Z0-9_-]+)$/, - ); - if (match) { - const scene = await loadScene(match[1], match[2]); - this.syncActionResult(scene); - } else { - const scene = await loadScene(null); - this.syncActionResult(scene); - } } + + const jsonMatch = window.location.hash.match( + /^#json=([0-9]+),([a-zA-Z0-9_-]+)$/, + ); + if (jsonMatch) { + const scene = await loadScene(jsonMatch[1], jsonMatch[2]); + this.syncActionResult(scene); + return; + } + + const roomMatch = getCollaborationLinkData(window.location.href); + if (roomMatch) { + this.initializeSocketClient(); + return; + } + const scene = await loadScene(null); + this.syncActionResult(scene); } public componentWillUnmount() { @@ -720,6 +834,12 @@ export class App extends React.Component { private handleCanvasPointerMove = ( event: React.PointerEvent, ) => { + const pointerCoords = viewportCoordsToSceneCoords( + event, + this.state, + this.canvas, + ); + this.savePointer(pointerCoords); gesture.pointers.set(event.pointerId, { x: event.clientX, y: event.clientY, @@ -1850,11 +1970,43 @@ export class App extends React.Component { } } + private savePointer = (pointerCoords: { x: number; y: number }) => { + if (isNaN(pointerCoords.x) || isNaN(pointerCoords.y)) { + // sometimes the pointer goes off screen + return; + } + this.socket && + this.broadcastSocketData({ + type: "MOUSE_LOCATION", + payload: { + socketID: this.socket.id, + pointerCoords, + }, + }); + }; + private saveDebounced = debounce(() => { saveToLocalStorage(elements, this.state); }, 300); componentDidUpdate() { + if (this.state.isCollaborating && !this.socket) { + this.initializeSocketClient(); + } + const pointerViewportCoords: { + [id: string]: { x: number; y: number }; + } = {}; + for (const clientId in this.state.remotePointers) { + const remotePointerCoord = this.state.remotePointers[clientId]; + pointerViewportCoords[clientId] = sceneCoordsToViewportCoords( + { + sceneX: remotePointerCoord.x, + sceneY: remotePointerCoord.y, + }, + this.state, + this.canvas, + ); + } const { atLeastOneVisibleElement, scrollBars } = renderScene( elements, this.state, @@ -1866,6 +2018,7 @@ export class App extends React.Component { scrollY: this.state.scrollY, viewBackgroundColor: this.state.viewBackgroundColor, zoom: this.state.zoom, + remotePointerViewportCoords: pointerViewportCoords, }, { renderOptimizations: true, @@ -1880,6 +2033,13 @@ export class App extends React.Component { } this.saveDebounced(); if (history.isRecording()) { + this.broadcastSocketData({ + type: "SCENE_UPDATE", + payload: { + elements, + appState: this.state, + }, + }); history.pushEntry(this.state, elements); history.skipRecording(); } diff --git a/src/components/icons.tsx b/src/components/icons.tsx index 1f43e572..376b1720 100644 --- a/src/components/icons.tsx +++ b/src/components/icons.tsx @@ -50,6 +50,12 @@ export const clipboard = createIcon( 512, ); +export const broadcast = createIcon( + "M150.94 192h33.73c11.01 0 18.61-10.83 14.86-21.18-4.93-13.58-7.55-27.98-7.55-42.82s2.62-29.24 7.55-42.82C203.29 74.83 195.68 64 184.67 64h-33.73c-7.01 0-13.46 4.49-15.41 11.23C130.64 92.21 128 109.88 128 128c0 18.12 2.64 35.79 7.54 52.76 1.94 6.74 8.39 11.24 15.4 11.24zM89.92 23.34C95.56 12.72 87.97 0 75.96 0H40.63c-6.27 0-12.14 3.59-14.74 9.31C9.4 45.54 0 85.65 0 128c0 24.75 3.12 68.33 26.69 118.86 2.62 5.63 8.42 9.14 14.61 9.14h34.84c12.02 0 19.61-12.74 13.95-23.37-49.78-93.32-16.71-178.15-.17-209.29zM614.06 9.29C611.46 3.58 605.6 0 599.33 0h-35.42c-11.98 0-19.66 12.66-14.02 23.25 18.27 34.29 48.42 119.42.28 209.23-5.72 10.68 1.8 23.52 13.91 23.52h35.23c6.27 0 12.13-3.58 14.73-9.29C630.57 210.48 640 170.36 640 128s-9.42-82.48-25.94-118.71zM489.06 64h-33.73c-11.01 0-18.61 10.83-14.86 21.18 4.93 13.58 7.55 27.98 7.55 42.82s-2.62 29.24-7.55 42.82c-3.76 10.35 3.85 21.18 14.86 21.18h33.73c7.02 0 13.46-4.49 15.41-11.24 4.9-16.97 7.53-34.64 7.53-52.76 0-18.12-2.64-35.79-7.54-52.76-1.94-6.75-8.39-11.24-15.4-11.24zm-116.3 100.12c7.05-10.29 11.2-22.71 11.2-36.12 0-35.35-28.63-64-63.96-64-35.32 0-63.96 28.65-63.96 64 0 13.41 4.15 25.83 11.2 36.12l-130.5 313.41c-3.4 8.15.46 17.52 8.61 20.92l29.51 12.31c8.15 3.4 17.52-.46 20.91-8.61L244.96 384h150.07l49.2 118.15c3.4 8.16 12.76 12.01 20.91 8.61l29.51-12.31c8.15-3.4 12-12.77 8.61-20.92l-130.5-313.41zM271.62 320L320 203.81 368.38 320h-96.76z", + 640, + 512, +); + export const trash = createIcon( "M32 464a48 48 0 0 0 48 48h288a48 48 0 0 0 48-48V128H32zm272-256a16 16 0 0 1 32 0v224a16 16 0 0 1-32 0zm-96 0a16 16 0 0 1 32 0v224a16 16 0 0 1-32 0zm-96 0a16 16 0 0 1 32 0v224a16 16 0 0 1-32 0zM432 32H312l-9.4-18.7A24 24 0 0 0 281.1 0H166.8a23.72 23.72 0 0 0-21.4 13.3L136 32H16A16 16 0 0 0 0 48v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16z", 448, diff --git a/src/data/index.ts b/src/data/index.ts index d857b11e..d43ee376 100644 --- a/src/data/index.ts +++ b/src/data/index.ts @@ -23,11 +23,145 @@ const BACKEND_GET = "https://json.excalidraw.com/api/v1/"; const BACKEND_V2_POST = "https://json.excalidraw.com/api/v2/post/"; const BACKEND_V2_GET = "https://json.excalidraw.com/api/v2/"; +export const SOCKET_SERVER = "https://excalidraw-socket.herokuapp.com"; + +export type EncryptedData = { + data: ArrayBuffer; + iv: Uint8Array; +}; + +export type SocketUpdateData = + | { + type: "SCENE_UPDATE"; + payload: { + elements: readonly ExcalidrawElement[]; + appState: AppState | null; + }; + } + | { + type: "MOUSE_LOCATION"; + payload: { + socketID: string; + pointerCoords: { x: number; y: number }; + }; + } + | { + type: "INVALID_RESPONSE"; + }; + // TODO: Defined globally, since file handles aren't yet serializable. // Once `FileSystemFileHandle` can be serialized, make this // part of `AppState`. (window as any).handle = null; +function byteToHex(byte: number): string { + return `0${byte.toString(16)}`.slice(-2); +} + +async function generateRandomID() { + const arr = new Uint8Array(10); + window.crypto.getRandomValues(arr); + return Array.from(arr, byteToHex).join(""); +} + +async function generateEncryptionKey() { + const key = await window.crypto.subtle.generateKey( + { + name: "AES-GCM", + length: 128, + }, + true, // extractable + ["encrypt", "decrypt"], + ); + return (await window.crypto.subtle.exportKey("jwk", key)).k; +} + +function createIV() { + const arr = new Uint8Array(12); + return window.crypto.getRandomValues(arr); +} + +export function getCollaborationLinkData(link: string) { + if (link.length === 0) { + return; + } + const hash = new URL(link).hash; + return hash.match(/^#room=([a-zA-Z0-9_-]+),([a-zA-Z0-9_-]+)$/); +} + +export async function generateCollaborationLink() { + const id = await generateRandomID(); + const key = await generateEncryptionKey(); + return `${window.location.href}#room=${id},${key}`; +} + +async function getImportedKey(key: string, usage: string): Promise { + return await window.crypto.subtle.importKey( + "jwk", + { + alg: "A128GCM", + ext: true, + k: key, + key_ops: ["encrypt", "decrypt"], + kty: "oct", + }, + { + name: "AES-GCM", + length: 128, + }, + false, // extractable + [usage], + ); +} + +export async function encryptAESGEM( + data: Uint8Array, + key: string, +): Promise { + const importedKey = await getImportedKey(key, "encrypt"); + const iv = createIV(); + return { + data: await window.crypto.subtle.encrypt( + { + name: "AES-GCM", + iv, + }, + importedKey, + data, + ), + iv, + }; +} + +export async function decryptAESGEM( + data: ArrayBuffer, + key: string, + iv: Uint8Array, +): Promise { + try { + const importedKey = await getImportedKey(key, "decrypt"); + const decrypted = await window.crypto.subtle.decrypt( + { + name: "AES-GCM", + iv: iv, + }, + importedKey, + data, + ); + + const decodedData = new TextDecoder("utf-8").decode( + new Uint8Array(decrypted) as any, + ); + return JSON.parse(decodedData); + } catch (error) { + window.alert(t("alerts.decryptFailed")); + console.error(error); + } + return { + type: "INVALID_RESPONSE", + }; +} + export async function exportToBackend( elements: readonly ExcalidrawElement[], appState: AppState, @@ -101,22 +235,7 @@ export async function importFromBackend( let data; if (privateKey) { const buffer = await response.arrayBuffer(); - const key = await window.crypto.subtle.importKey( - "jwk", - { - alg: "A128GCM", - ext: true, - k: privateKey, - key_ops: ["encrypt", "decrypt"], - kty: "oct", - }, - { - name: "AES-GCM", - length: 128, - }, - false, // extractable - ["decrypt"], - ); + const key = await getImportedKey(privateKey, "decrypt"); const iv = new Uint8Array(12); const decrypted = await window.crypto.subtle.decrypt( { diff --git a/src/data/localStorage.ts b/src/data/localStorage.ts index 1b5fc09b..59d1a523 100644 --- a/src/data/localStorage.ts +++ b/src/data/localStorage.ts @@ -34,6 +34,9 @@ export function restoreFromLocalStorage() { if (savedState) { try { appState = JSON.parse(savedState) as AppState; + // If we're retrieving from local storage, we should not be collaborating + appState.isCollaborating = false; + appState.collaboratorCount = 0; } catch { // Do nothing because appState is already null } diff --git a/src/locales/en.json b/src/locales/en.json index f7a47887..c9a22b81 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -70,6 +70,7 @@ "importBackendFailed": "Importing from backend failed.", "cannotExportEmptyCanvas": "Cannot export empty canvas.", "couldNotCopyToClipboard": "Couldn't copy to clipboard. Try using Chrome browser.", + "decryptFailed": "Couldn't decrypt data.", "uploadedSecurly": "The upload has been secured with end-to-end encryption, which means that Excalidraw server and third parties can't read the content." }, "toolBar": { diff --git a/src/renderer/renderScene.ts b/src/renderer/renderScene.ts index b95b050f..80d13f28 100644 --- a/src/renderer/renderScene.ts +++ b/src/renderer/renderScene.ts @@ -172,6 +172,15 @@ export function renderScene( } } + // Paint remote pointers + for (const clientId in sceneState.remotePointerViewportCoords) { + const { x, y } = sceneState.remotePointerViewportCoords[clientId]; + context.beginPath(); + context.arc(x, y, 5, 0, 2 * Math.PI); + context.fill(); + context.stroke(); + } + // Paint scrollbars if (renderScrollbars) { const scrollBars = getScrollBars( diff --git a/src/scene/export.ts b/src/scene/export.ts index ae47b721..e1f32998 100644 --- a/src/scene/export.ts +++ b/src/scene/export.ts @@ -49,6 +49,7 @@ export function exportToCanvas( scrollX: normalizeScroll(-minX + exportPadding), scrollY: normalizeScroll(-minY + exportPadding), zoom: 1, + remotePointerViewportCoords: {}, }, { renderScrollbars: false, diff --git a/src/scene/types.ts b/src/scene/types.ts index e744e6cb..cd48c4ab 100644 --- a/src/scene/types.ts +++ b/src/scene/types.ts @@ -7,6 +7,7 @@ export type SceneState = { // null indicates transparent bg viewBackgroundColor: string | null; zoom: number; + remotePointerViewportCoords: { [id: string]: { x: number; y: number } }; }; export type SceneScroll = { diff --git a/src/types.ts b/src/types.ts index 9dec663d..457d806f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -29,11 +29,14 @@ export type AppState = { scrolledOutside: boolean; name: string; selectedId?: string; + isCollaborating: boolean; isResizing: boolean; zoom: number; openMenu: "canvas" | "shape" | null; lastPointerDownWith: PointerType; selectedElementIds: { [id: string]: boolean }; + remotePointers: { [id: string]: { x: number; y: number } }; + collaboratorCount: number; }; export type PointerCoords = Readonly<{