diff --git a/src/element/collision.ts b/src/element/collision.ts
index abcf2652..01ea17e5 100644
--- a/src/element/collision.ts
+++ b/src/element/collision.ts
@@ -8,6 +8,10 @@ import {
   getLinePoints
 } from "./bounds";
 
+function isElementDraggableFromInside(element: ExcalidrawElement): boolean {
+  return element.backgroundColor !== "transparent" || element.isSelected;
+}
+
 export function hitTest(
   element: ExcalidrawElement,
   x: number,
@@ -51,7 +55,7 @@ export function hitTest(
       ty /= t;
     });
 
-    if (element.backgroundColor !== "transparent") {
+    if (isElementDraggableFromInside(element)) {
       return (
         a * tx - (px - lineThreshold) >= 0 && b * ty - (py - lineThreshold) >= 0
       );
@@ -61,7 +65,7 @@ export function hitTest(
   } else if (element.type === "rectangle") {
     const [x1, y1, x2, y2] = getElementAbsoluteCoords(element);
 
-    if (element.backgroundColor !== "transparent") {
+    if (isElementDraggableFromInside(element)) {
       return (
         x > x1 - lineThreshold &&
         x < x2 + lineThreshold &&
@@ -94,7 +98,7 @@ export function hitTest(
       leftY
     ] = getDiamondPoints(element);
 
-    if (element.backgroundColor !== "transparent") {
+    if (isElementDraggableFromInside(element)) {
       // TODO: remove this when we normalize coordinates globally
       if (topY > bottomY) [bottomY, topY] = [topY, bottomY];
       if (rightX < leftX) [leftX, rightX] = [rightX, leftX];