diff --git a/src/index.tsx b/src/index.tsx
index 8be8e88747..e19487c3ed 100644
--- a/src/index.tsx
+++ b/src/index.tsx
@@ -15,7 +15,7 @@ import {
 import {
   clearSelection,
   deleteSelectedElements,
-  getElementsWithinSelection,
+  setSelection,
   isOverScrollBars,
   restoreFromLocalStorage,
   saveToLocalStorage,
@@ -746,24 +746,13 @@ export class App extends React.Component<{}, AppState> {
                 this.state.scrollY;
               draggingElement.width = width;
               // Make a perfect square or circle when shift is enabled
-              draggingElement.height =
-                // Shift key on selection must add items to selection
-                e.shiftKey && this.state.elementType !== "selection"
-                  ? Math.abs(width) * Math.sign(height)
-                  : height;
+              draggingElement.height = e.shiftKey
+                ? Math.abs(width) * Math.sign(height)
+                : height;
               draggingElement.shape = null;
 
               if (this.state.elementType === "selection") {
-                const elementsWithinSelection = getElementsWithinSelection(
-                  elements,
-                  draggingElement
-                );
-                if (!e.shiftKey) {
-                  elements = clearSelection(elements);
-                }
-                elementsWithinSelection.forEach(
-                  element => (element.isSelected = true)
-                );
+                elements = setSelection(elements, draggingElement);
               }
               // We don't want to save history when moving an element
               history.skipRecording();
diff --git a/src/scene/index.ts b/src/scene/index.ts
index 21e8969083..e812a8271b 100644
--- a/src/scene/index.ts
+++ b/src/scene/index.ts
@@ -4,7 +4,7 @@ export {
   getSelectedIndices,
   deleteSelectedElements,
   someElementIsSelected,
-  getElementsWithinSelection,
+  setSelection,
   getSelectedAttribute
 } from "./selection";
 export {
diff --git a/src/scene/selection.ts b/src/scene/selection.ts
index 782f3014f4..6347fae8ce 100644
--- a/src/scene/selection.ts
+++ b/src/scene/selection.ts
@@ -1,7 +1,7 @@
 import { ExcalidrawElement } from "../element/types";
 import { getElementAbsoluteCoords } from "../element";
 
-export function getElementsWithinSelection(
+export function setSelection(
   elements: readonly ExcalidrawElement[],
   selection: ExcalidrawElement
 ) {
@@ -11,22 +11,22 @@ export function getElementsWithinSelection(
     selectionX2,
     selectionY2
   ] = getElementAbsoluteCoords(selection);
-
-  return elements.filter(element => {
+  elements.forEach(element => {
     const [
       elementX1,
       elementY1,
       elementX2,
       elementY2
     ] = getElementAbsoluteCoords(element);
-    return (
+    element.isSelected =
       element.type !== "selection" &&
       selectionX1 <= elementX1 &&
       selectionY1 <= elementY1 &&
       selectionX2 >= elementX2 &&
-      selectionY2 >= elementY2
-    );
+      selectionY2 >= elementY2;
   });
+
+  return elements;
 }
 
 export function clearSelection(elements: readonly ExcalidrawElement[]) {