diff --git a/src/index.tsx b/src/index.tsx
index e19487c3..8be8e887 100644
--- a/src/index.tsx
+++ b/src/index.tsx
@@ -15,7 +15,7 @@ import {
 import {
   clearSelection,
   deleteSelectedElements,
-  setSelection,
+  getElementsWithinSelection,
   isOverScrollBars,
   restoreFromLocalStorage,
   saveToLocalStorage,
@@ -746,13 +746,24 @@ export class App extends React.Component<{}, AppState> {
                 this.state.scrollY;
               draggingElement.width = width;
               // Make a perfect square or circle when shift is enabled
-              draggingElement.height = e.shiftKey
-                ? Math.abs(width) * Math.sign(height)
-                : height;
+              draggingElement.height =
+                // Shift key on selection must add items to selection
+                e.shiftKey && this.state.elementType !== "selection"
+                  ? Math.abs(width) * Math.sign(height)
+                  : height;
               draggingElement.shape = null;
 
               if (this.state.elementType === "selection") {
-                elements = setSelection(elements, draggingElement);
+                const elementsWithinSelection = getElementsWithinSelection(
+                  elements,
+                  draggingElement
+                );
+                if (!e.shiftKey) {
+                  elements = clearSelection(elements);
+                }
+                elementsWithinSelection.forEach(
+                  element => (element.isSelected = true)
+                );
               }
               // We don't want to save history when moving an element
               history.skipRecording();
diff --git a/src/scene/index.ts b/src/scene/index.ts
index e812a827..21e89690 100644
--- a/src/scene/index.ts
+++ b/src/scene/index.ts
@@ -4,7 +4,7 @@ export {
   getSelectedIndices,
   deleteSelectedElements,
   someElementIsSelected,
-  setSelection,
+  getElementsWithinSelection,
   getSelectedAttribute
 } from "./selection";
 export {
diff --git a/src/scene/selection.ts b/src/scene/selection.ts
index 6347fae8..782f3014 100644
--- a/src/scene/selection.ts
+++ b/src/scene/selection.ts
@@ -1,7 +1,7 @@
 import { ExcalidrawElement } from "../element/types";
 import { getElementAbsoluteCoords } from "../element";
 
-export function setSelection(
+export function getElementsWithinSelection(
   elements: readonly ExcalidrawElement[],
   selection: ExcalidrawElement
 ) {
@@ -11,22 +11,22 @@ export function setSelection(
     selectionX2,
     selectionY2
   ] = getElementAbsoluteCoords(selection);
-  elements.forEach(element => {
+
+  return elements.filter(element => {
     const [
       elementX1,
       elementY1,
       elementX2,
       elementY2
     ] = getElementAbsoluteCoords(element);
-    element.isSelected =
+    return (
       element.type !== "selection" &&
       selectionX1 <= elementX1 &&
       selectionY1 <= elementY1 &&
       selectionX2 >= elementX2 &&
-      selectionY2 >= elementY2;
+      selectionY2 >= elementY2
+    );
   });
-
-  return elements;
 }
 
 export function clearSelection(elements: readonly ExcalidrawElement[]) {