diff --git a/src/components/Dialog.tsx b/src/components/Dialog.tsx
index f2c917db..ada1bb5a 100644
--- a/src/components/Dialog.tsx
+++ b/src/components/Dialog.tsx
@@ -1,5 +1,5 @@
 import clsx from "clsx";
-import React, { useEffect } from "react";
+import React, { useEffect, useState } from "react";
 import { useCallbackRefState } from "../hooks/useCallbackRefState";
 import { t } from "../i18n";
 import { useIsMobile } from "../components/App";
@@ -20,6 +20,8 @@ export const Dialog = (props: {
   theme?: AppState["theme"];
 }) => {
   const [islandNode, setIslandNode] = useCallbackRefState<HTMLDivElement>();
+  const [lastActiveElement] = useState(document.activeElement);
+
   useEffect(() => {
     if (!islandNode) {
       return;
@@ -66,12 +68,17 @@ export const Dialog = (props: {
     return focusableElements ? Array.from(focusableElements) : [];
   };
 
+  const onClose = () => {
+    (lastActiveElement as HTMLElement).focus();
+    props.onCloseRequest();
+  };
+
   return (
     <Modal
       className={clsx("Dialog", props.className)}
       labelledBy="dialog-title"
       maxWidth={props.small ? 550 : 800}
-      onCloseRequest={props.onCloseRequest}
+      onCloseRequest={onClose}
       theme={props.theme}
     >
       <Island ref={setIslandNode}>
@@ -79,7 +86,7 @@ export const Dialog = (props: {
           <span className="Dialog__titleContent">{props.title}</span>
           <button
             className="Modal__close"
-            onClick={props.onCloseRequest}
+            onClick={onClose}
             aria-label={t("buttons.close")}
           >
             {useIsMobile() ? back : close}
diff --git a/src/components/ErrorDialog.tsx b/src/components/ErrorDialog.tsx
index 63493f80..f383a00c 100644
--- a/src/components/ErrorDialog.tsx
+++ b/src/components/ErrorDialog.tsx
@@ -2,6 +2,7 @@ import React, { useState } from "react";
 import { t } from "../i18n";
 
 import { Dialog } from "./Dialog";
+import { useExcalidrawContainer } from "./App";
 
 export const ErrorDialog = ({
   message,
@@ -11,6 +12,7 @@ export const ErrorDialog = ({
   onClose?: () => void;
 }) => {
   const [modalIsShown, setModalIsShown] = useState(!!message);
+  const excalidrawContainer = useExcalidrawContainer();
 
   const handleClose = React.useCallback(() => {
     setModalIsShown(false);
@@ -18,8 +20,9 @@ export const ErrorDialog = ({
     if (onClose) {
       onClose();
     }
-    document.querySelector<HTMLElement>(".excalidraw-container")?.focus();
-  }, [onClose]);
+    // TODO: Fix the A11y issues so this is never needed since we should always focus on last active element
+    excalidrawContainer?.focus();
+  }, [onClose, excalidrawContainer]);
 
   return (
     <>
diff --git a/src/components/ExportDialog.tsx b/src/components/ExportDialog.tsx
index f09ed99d..23abb8c6 100644
--- a/src/components/ExportDialog.tsx
+++ b/src/components/ExportDialog.tsx
@@ -244,11 +244,9 @@ export const ExportDialog = ({
   onExportToBackend?: ExportCB;
 }) => {
   const [modalIsShown, setModalIsShown] = useState(false);
-  const triggerButton = useRef<HTMLButtonElement>(null);
 
   const handleClose = React.useCallback(() => {
     setModalIsShown(false);
-    triggerButton.current?.focus();
   }, []);
 
   return (
@@ -263,7 +261,6 @@ export const ExportDialog = ({
         aria-label={t("buttons.export")}
         showAriaLabel={useIsMobile()}
         title={t("buttons.export")}
-        ref={triggerButton}
       />
       {modalIsShown && (
         <Dialog onCloseRequest={handleClose} title={t("buttons.export")}>
diff --git a/src/components/LayerUI.tsx b/src/components/LayerUI.tsx
index 67b579d4..df521361 100644
--- a/src/components/LayerUI.tsx
+++ b/src/components/LayerUI.tsx
@@ -671,10 +671,6 @@ const LayerUI = ({
       {appState.showHelpDialog && (
         <HelpDialog
           onClose={() => {
-            const helpIcon = document.querySelector(
-              ".help-icon",
-            )! as HTMLElement;
-            helpIcon.focus();
             setAppState({ showHelpDialog: false });
           }}
         />
diff --git a/src/excalidraw-app/collab/CollabWrapper.tsx b/src/excalidraw-app/collab/CollabWrapper.tsx
index a029b043..8fe753f8 100644
--- a/src/excalidraw-app/collab/CollabWrapper.tsx
+++ b/src/excalidraw-app/collab/CollabWrapper.tsx
@@ -591,8 +591,6 @@ class CollabWrapper extends PureComponent<Props, CollabState> {
 
   handleClose = () => {
     this.setState({ modalIsShown: false });
-    const collabIcon = document.querySelector(".CollabButton") as HTMLElement;
-    collabIcon.focus();
   };
 
   onUsernameChange = (username: string) => {