diff --git a/src/actions/actionCanvas.tsx b/src/actions/actionCanvas.tsx
index a927c5af..2e56954e 100644
--- a/src/actions/actionCanvas.tsx
+++ b/src/actions/actionCanvas.tsx
@@ -26,7 +26,7 @@ export const actionChangeViewBackgroundColor = register({
       commitToHistory: !!value.viewBackgroundColor,
     };
   },
-  PanelComponent: ({ appState, updateData }) => {
+  PanelComponent: ({ elements, appState, updateData }) => {
     return (
       <div style={{ position: "relative" }}>
         <ColorPicker
@@ -39,6 +39,8 @@ export const actionChangeViewBackgroundColor = register({
             updateData({ openPopup: active ? "canvasColorPicker" : null })
           }
           data-testid="canvas-background-picker"
+          elements={elements}
+          appState={appState}
         />
       </div>
     );
diff --git a/src/actions/actionProperties.tsx b/src/actions/actionProperties.tsx
index 87214b32..203c6257 100644
--- a/src/actions/actionProperties.tsx
+++ b/src/actions/actionProperties.tsx
@@ -233,6 +233,8 @@ export const actionChangeStrokeColor = register({
         setActive={(active) =>
           updateData({ openPopup: active ? "strokeColorPicker" : null })
         }
+        elements={elements}
+        appState={appState}
       />
     </>
   ),
@@ -273,6 +275,8 @@ export const actionChangeBackgroundColor = register({
         setActive={(active) =>
           updateData({ openPopup: active ? "backgroundColorPicker" : null })
         }
+        elements={elements}
+        appState={appState}
       />
     </>
   ),
diff --git a/src/components/ColorPicker.scss b/src/components/ColorPicker.scss
index c5a178d3..fdcb9baa 100644
--- a/src/components/ColorPicker.scss
+++ b/src/components/ColorPicker.scss
@@ -46,7 +46,7 @@
     top: -11px;
   }
 
-  .color-picker-content {
+  .color-picker-content--default {
     padding: 0.5rem;
     display: grid;
     grid-template-columns: repeat(5, auto);
@@ -59,6 +59,26 @@
     }
   }
 
+  .color-picker-content--canvas {
+    display: flex;
+    flex-direction: column;
+    padding: 0.25rem;
+
+    &-title {
+      color: $oc-gray-6;
+      font-size: 12px;
+      padding: 0 0.25rem;
+    }
+
+    &-colors {
+      padding: 0.5rem 0;
+
+      .color-picker-swatch {
+        margin: 0 0.25rem;
+      }
+    }
+  }
+
   .color-picker-content .color-input-container {
     grid-column: 1 / span 5;
   }
diff --git a/src/components/ColorPicker.tsx b/src/components/ColorPicker.tsx
index 16eed736..e409557a 100644
--- a/src/components/ColorPicker.tsx
+++ b/src/components/ColorPicker.tsx
@@ -7,6 +7,53 @@ import { isArrowKey, KEYS } from "../keys";
 import { t, getLanguage } from "../i18n";
 import { isWritableElement } from "../utils";
 import colors from "../colors";
+import { ExcalidrawElement } from "../element/types";
+import { AppState } from "../types";
+
+const MAX_CUSTOM_COLORS = 5;
+const MAX_DEFAULT_COLORS = 15;
+
+export const getCustomColors = (
+  elements: readonly ExcalidrawElement[],
+  type: "elementBackground" | "elementStroke",
+) => {
+  const customColors: string[] = [];
+  const updatedElements = elements
+    .filter((element) => !element.isDeleted)
+    .sort((ele1, ele2) => ele2.updated - ele1.updated);
+
+  let index = 0;
+  const elementColorTypeMap = {
+    elementBackground: "backgroundColor",
+    elementStroke: "strokeColor",
+  };
+  const colorType = elementColorTypeMap[type] as
+    | "backgroundColor"
+    | "strokeColor";
+  while (
+    index < updatedElements.length &&
+    customColors.length < MAX_CUSTOM_COLORS
+  ) {
+    const element = updatedElements[index];
+
+    if (
+      customColors.length < MAX_CUSTOM_COLORS &&
+      isCustomColor(element[colorType], type) &&
+      !customColors.includes(element[colorType])
+    ) {
+      customColors.push(element[colorType]);
+    }
+    index++;
+  }
+  return customColors;
+};
+
+const isCustomColor = (
+  color: string,
+  type: "elementBackground" | "elementStroke",
+) => {
+  return !colors[type].includes(color);
+};
 
 const isValidColor = (color: string) => {
   const style = new Option().style;
@@ -35,6 +82,7 @@ const keyBindings = [
   ["1", "2", "3", "4", "5"],
   ["q", "w", "e", "r", "t"],
   ["a", "s", "d", "f", "g"],
+  ["z", "x", "c", "v", "b"],
 ].flat();
 
 const Picker = ({
@@ -45,6 +93,7 @@ const Picker = ({
   label,
   showInput = true,
   type,
+  elements,
 }: {
   colors: string[];
   color: string | null;
@@ -53,12 +102,20 @@ const Picker = ({
   label: string;
   showInput: boolean;
   type: "canvasBackground" | "elementBackground" | "elementStroke";
+  elements: readonly ExcalidrawElement[];
 }) => {
   const firstItem = React.useRef<HTMLButtonElement>();
   const activeItem = React.useRef<HTMLButtonElement>();
   const gallery = React.useRef<HTMLDivElement>();
   const colorInput = React.useRef<HTMLInputElement>();
 
+  const [customColors] = React.useState(() => {
+    if (type === "canvasBackground") {
+      return [];
+    }
+    return getCustomColors(elements, type);
+  });
+
   React.useEffect(() => {
     // After the component is first mounted focus on first input
     if (activeItem.current) {
@@ -85,23 +142,42 @@ const Picker = ({
     } else if (isArrowKey(event.key)) {
       const { activeElement } = document;
       const isRTL = getLanguage().rtl;
-      const index = Array.prototype.indexOf.call(
-        gallery!.current!.children,
+      let isCustom = false;
+      let index = Array.prototype.indexOf.call(
+        gallery!.current!.querySelector(".color-picker-content--default")!
+          .children,
         activeElement,
       );
+      if (index === -1) {
+        index = Array.prototype.indexOf.call(
+          gallery!.current!.querySelector(
+            ".color-picker-content--canvas-colors",
+          )!.children,
+          activeElement,
+        );
+        if (index !== -1) {
+          isCustom = true;
+        }
+      }
+      const parentSelector = isCustom
+        ? gallery!.current!.querySelector(
+            ".color-picker-content--canvas-colors",
+          )!
+        : gallery!.current!.querySelector(".color-picker-content--default")!;
+
       if (index !== -1) {
-        const length = gallery!.current!.children.length - (showInput ? 1 : 0);
+        const length = parentSelector!.children.length - (showInput ? 1 : 0);
         const nextIndex =
           event.key === (isRTL ? KEYS.ARROW_LEFT : KEYS.ARROW_RIGHT)
             ? (index + 1) % length
             : event.key === (isRTL ? KEYS.ARROW_RIGHT : KEYS.ARROW_LEFT)
             ? (length + index - 1) % length
-            : event.key === KEYS.ARROW_DOWN
+            : !isCustom && event.key === KEYS.ARROW_DOWN
             ? (index + 5) % length
-            : event.key === KEYS.ARROW_UP
+            : !isCustom && event.key === KEYS.ARROW_UP
             ? (length + index - 5) % length
             : index;
-        (gallery!.current!.children![nextIndex] as any).focus();
+        (parentSelector!.children![nextIndex] as HTMLElement)?.focus();
       }
       event.preventDefault();
     } else if (
@@ -109,7 +185,15 @@ const Picker = ({
       !isWritableElement(event.target)
     ) {
       const index = keyBindings.indexOf(event.key.toLowerCase());
-      (gallery!.current!.children![index] as any).focus();
+      const isCustom = index >= MAX_DEFAULT_COLORS;
+      const parentSelector = isCustom
+        ? gallery!.current!.querySelector(
+            ".color-picker-content--canvas-colors",
+          )!
+        : gallery!.current!.querySelector(".color-picker-content--default")!;
+      const actualIndex = isCustom ? index - MAX_DEFAULT_COLORS : index;
+      (parentSelector!.children![actualIndex] as HTMLElement)?.focus();
+
       event.preventDefault();
     } else if (event.key === KEYS.ESCAPE || event.key === KEYS.ENTER) {
       event.preventDefault();
@@ -119,6 +203,50 @@ const Picker = ({
     event.stopPropagation();
   };
 
+  const renderColors = (colors: Array<string>, custom: boolean = false) => {
+    return colors.map((_color, i) => {
+      const _colorWithoutHash = _color.replace("#", "");
+      const keyBinding = custom
+        ? keyBindings[i + MAX_DEFAULT_COLORS]
+        : keyBindings[i];
+      const label = custom
+        ? _colorWithoutHash
+        : t(`colors.${_colorWithoutHash}`);
+      return (
+        <button
+          className="color-picker-swatch"
+          onClick={(event) => {
+            (event.currentTarget as HTMLButtonElement).focus();
+            onChange(_color);
+          }}
+          title={`${label}${
+            !isTransparent(_color) ? ` (${_color})` : ""
+          } — ${keyBinding.toUpperCase()}`}
+          aria-label={label}
+          aria-keyshortcuts={keyBindings[i]}
+          style={{ color: _color }}
+          key={_color}
+          ref={(el) => {
+            if (!custom && el && i === 0) {
+              firstItem.current = el;
+            }
+            if (el && _color === color) {
+              activeItem.current = el;
+            }
+          }}
+          onFocus={() => {
+            onChange(_color);
+          }}
+        >
+          {isTransparent(_color) ? (
+            <div className="color-picker-transparent"></div>
+          ) : undefined}
+          <span className="color-picker-keybinding">{keyBinding}</span>
+        </button>
+      );
+    });
+  };
+
   return (
     <div
       className={`color-picker color-picker-type-${type}`}
@@ -138,41 +266,20 @@ const Picker = ({
         }}
         tabIndex={0}
       >
-        {colors.map((_color, i) => {
-          const _colorWithoutHash = _color.replace("#", "");
-          return (
-            <button
-              className="color-picker-swatch"
-              onClick={(event) => {
-                (event.currentTarget as HTMLButtonElement).focus();
-                onChange(_color);
-              }}
-              title={`${t(`colors.${_colorWithoutHash}`)}${
-                !isTransparent(_color) ? ` (${_color})` : ""
-              } — ${keyBindings[i].toUpperCase()}`}
-              aria-label={t(`colors.${_colorWithoutHash}`)}
-              aria-keyshortcuts={keyBindings[i]}
-              style={{ color: _color }}
-              key={_color}
-              ref={(el) => {
-                if (el && i === 0) {
-                  firstItem.current = el;
-                }
-                if (el && _color === color) {
-                  activeItem.current = el;
-                }
-              }}
-              onFocus={() => {
-                onChange(_color);
-              }}
-            >
-              {isTransparent(_color) ? (
-                <div className="color-picker-transparent"></div>
-              ) : undefined}
-              <span className="color-picker-keybinding">{keyBindings[i]}</span>
-            </button>
-          );
-        })}
+        <div className="color-picker-content--default">
+          {renderColors(colors)}
+        </div>
+        {!!customColors.length && (
+          <div className="color-picker-content--canvas">
+            <span className="color-picker-content--canvas-title">
+              {t("labels.canvasColors")}
+            </span>
+            <div className="color-picker-content--canvas-colors">
+              {renderColors(customColors, true)}
+            </div>
+          </div>
+        )}
+
         {showInput && (
           <ColorInput
             color={color}
@@ -246,6 +353,8 @@ export const ColorPicker = ({
   label,
   isActive,
   setActive,
+  elements,
+  appState,
 }: {
   type: "canvasBackground" | "elementBackground" | "elementStroke";
   color: string | null;
@@ -253,6 +362,8 @@ export const ColorPicker = ({
   label: string;
   isActive: boolean;
   setActive: (active: boolean) => void;
+  elements: readonly ExcalidrawElement[];
+  appState: AppState;
 }) => {
   const pickerButton = React.useRef<HTMLButtonElement>(null);
 
@@ -294,6 +405,7 @@ export const ColorPicker = ({
               label={label}
               showInput={false}
               type={type}
+              elements={elements}
             />
           </Popover>
         ) : null}
diff --git a/src/locales/en.json b/src/locales/en.json
index fb01e1ba..b5595c2d 100644
--- a/src/locales/en.json
+++ b/src/locales/en.json
@@ -64,6 +64,7 @@
     "cartoonist": "Cartoonist",
     "fileTitle": "File name",
     "colorPicker": "Color picker",
+    "canvasColors": "Used on canvas",
     "canvasBackground": "Canvas background",
     "drawingCanvas": "Drawing canvas",
     "layers": "Layers",