From b91158198ec8175fbbda104cf12a8d0c38dd2a20 Mon Sep 17 00:00:00 2001 From: David Luzar Date: Sun, 6 Nov 2022 19:41:14 +0100 Subject: [PATCH] feat: clean unused images only after 24hrs (local-only) (#5839) * feat: clean unused images only after 24hrs (local-only) * fix test * make optional for now --- src/components/App.tsx | 1 + src/excalidraw-app/data/FileManager.ts | 1 + src/excalidraw-app/data/LocalData.ts | 40 ++++++++++++++++++++----- src/excalidraw-app/data/firebase.ts | 1 + src/packages/excalidraw/example/App.tsx | 1 + src/tests/export.test.tsx | 1 + src/types.ts | 11 +++++++ 7 files changed, 48 insertions(+), 8 deletions(-) diff --git a/src/components/App.tsx b/src/components/App.tsx index 3d5e6a36..9df3bb67 100644 --- a/src/components/App.tsx +++ b/src/components/App.tsx @@ -5243,6 +5243,7 @@ class App extends React.Component { id: fileId, dataURL, created: Date.now(), + lastRetrieved: Date.now(), }, }; const cachedImageData = this.imageCache.get(fileId); diff --git a/src/excalidraw-app/data/FileManager.ts b/src/excalidraw-app/data/FileManager.ts index 533387a6..419407a6 100644 --- a/src/excalidraw-app/data/FileManager.ts +++ b/src/excalidraw-app/data/FileManager.ts @@ -195,6 +195,7 @@ export const encodeFilesForUpload = async ({ id, mimeType: fileData.mimeType, created: Date.now(), + lastRetrieved: Date.now(), }, }); diff --git a/src/excalidraw-app/data/LocalData.ts b/src/excalidraw-app/data/LocalData.ts index 986c8eb8..08f91d8d 100644 --- a/src/excalidraw-app/data/LocalData.ts +++ b/src/excalidraw-app/data/LocalData.ts @@ -10,7 +10,7 @@ * (localStorage, indexedDB). */ -import { createStore, keys, del, getMany, set } from "idb-keyval"; +import { createStore, entries, del, getMany, set, setMany } from "idb-keyval"; import { clearAppStateForLocalStorage } from "../../appState"; import { clearElementsForLocalStorage } from "../../element"; import { ExcalidrawElement, FileId } from "../../element/types"; @@ -25,12 +25,21 @@ const filesStore = createStore("files-db", "files-store"); class LocalFileManager extends FileManager { clearObsoleteFiles = async (opts: { currentFileIds: FileId[] }) => { - const allIds = await keys(filesStore); - for (const id of allIds) { - if (!opts.currentFileIds.includes(id as FileId)) { - del(id, filesStore); + await entries(filesStore).then((entries) => { + for (const [id, imageData] of entries as [FileId, BinaryFileData][]) { + // if image is unused (not on canvas) & is older than 1 day, delete it + // from storage. We check `lastRetrieved` we care about the last time + // the image was used (loaded on canvas), not when it was initially + // created. + if ( + (!imageData.lastRetrieved || + Date.now() - imageData.lastRetrieved > 24 * 3600 * 1000) && + !opts.currentFileIds.includes(id as FileId) + ) { + del(id, filesStore); + } } - } + }); }; } @@ -111,18 +120,33 @@ export class LocalData { static fileStorage = new LocalFileManager({ getFiles(ids) { return getMany(ids, filesStore).then( - (filesData: (BinaryFileData | undefined)[]) => { + async (filesData: (BinaryFileData | undefined)[]) => { const loadedFiles: BinaryFileData[] = []; const erroredFiles = new Map(); + + const filesToSave: [FileId, BinaryFileData][] = []; + filesData.forEach((data, index) => { const id = ids[index]; if (data) { - loadedFiles.push(data); + const _data: BinaryFileData = { + ...data, + lastRetrieved: Date.now(), + }; + filesToSave.push([id, _data]); + loadedFiles.push(_data); } else { erroredFiles.set(id, true); } }); + try { + // save loaded files back to storage with updated `lastRetrieved` + setMany(filesToSave, filesStore); + } catch (error) { + console.warn(error); + } + return { loadedFiles, erroredFiles }; }, ); diff --git a/src/excalidraw-app/data/firebase.ts b/src/excalidraw-app/data/firebase.ts index 018bc3c4..b6b26249 100644 --- a/src/excalidraw-app/data/firebase.ts +++ b/src/excalidraw-app/data/firebase.ts @@ -330,6 +330,7 @@ export const loadFilesFromFirebase = async ( id, dataURL, created: metadata?.created || Date.now(), + lastRetrieved: metadata?.created || Date.now(), }); } else { erroredFiles.set(id, true); diff --git a/src/packages/excalidraw/example/App.tsx b/src/packages/excalidraw/example/App.tsx index 7608c32c..ee555f37 100644 --- a/src/packages/excalidraw/example/App.tsx +++ b/src/packages/excalidraw/example/App.tsx @@ -148,6 +148,7 @@ export default function App() { dataURL: reader.result as BinaryFileData["dataURL"], mimeType: MIME_TYPES.jpg, created: 1644915140367, + lastRetrieved: 1644915140367, }, ]; diff --git a/src/tests/export.test.tsx b/src/tests/export.test.tsx index 02e6a13a..da496dea 100644 --- a/src/tests/export.test.tsx +++ b/src/tests/export.test.tsx @@ -158,6 +158,7 @@ describe("export", () => { dataURL: await getDataURL(await API.loadFile("./fixtures/deer.png")), mimeType: "image/png", created: Date.now(), + lastRetrieved: Date.now(), }, } as const; diff --git a/src/types.ts b/src/types.ts index 52a84ef3..620f7b74 100644 --- a/src/types.ts +++ b/src/types.ts @@ -61,7 +61,18 @@ export type BinaryFileData = { | typeof MIME_TYPES.binary; id: FileId; dataURL: DataURL; + /** + * Epoch timestamp in milliseconds + */ created: number; + /** + * Indicates when the file was last retrieved from storage to be loaded + * onto the scene. We use this flag to determine whether to delete unused + * files from storage. + * + * Epoch timestamp in milliseconds. + */ + lastRetrieved?: number; }; export type BinaryFileMetadata = Omit;