diff --git a/src/components/App.tsx b/src/components/App.tsx
index 860c1b65..d680820f 100644
--- a/src/components/App.tsx
+++ b/src/components/App.tsx
@@ -99,6 +99,7 @@ import { mutateElement, newElementWith } from "../element/mutateElement";
 import { invalidateShapeForElement } from "../renderer/renderElement";
 import { unstable_batchedUpdates } from "react-dom";
 import { SceneStateCallbackRemover } from "../scene/globalScene";
+import { rescalePoints } from "../points";
 
 function withBatchedUpdates<
   TFunction extends ((event: any) => void) | (() => void)
@@ -1801,89 +1802,68 @@ export class App extends React.Component<any, AppState> {
               }
               break;
             case "n": {
-              let points;
-              if (element.points.length > 0) {
-                const len = element.points.length;
-                points = [...element.points].sort((a, b) => a[1] - b[1]) as [
-                  number,
-                  number,
-                ][];
-
-                for (let i = 1; i < points.length; ++i) {
-                  const pnt = points[i];
-                  pnt[1] -= deltaY / (len - i);
-                }
+              const height = element.height - deltaY;
+              if (height <= 0) {
+                break;
               }
 
               mutateElement(element, {
-                height: element.height - deltaY,
+                height,
                 y: element.y + deltaY,
-                points,
+                points:
+                  element.points.length > 0
+                    ? rescalePoints(1, height, element.points)
+                    : undefined,
               });
+
               break;
             }
             case "w": {
-              let points;
-              if (element.points.length > 0) {
-                const len = element.points.length;
-                points = [...element.points].sort((a, b) => a[0] - b[0]) as [
-                  number,
-                  number,
-                ][];
+              const width = element.width - deltaX;
 
-                for (let i = 0; i < points.length; ++i) {
-                  const pnt = points[i];
-                  pnt[0] -= deltaX / (len - i);
-                }
+              if (width <= 0) {
+                // Someday we should implement logic to flip the shape. But for now, just stop.
+                break;
               }
 
               mutateElement(element, {
-                width: element.width - deltaX,
+                width,
                 x: element.x + deltaX,
-                points,
+                points:
+                  element.points.length > 0
+                    ? rescalePoints(0, width, element.points)
+                    : undefined,
               });
               break;
             }
             case "s": {
-              let points;
-
-              if (element.points.length > 0) {
-                const len = element.points.length;
-                points = [...element.points].sort((a, b) => a[1] - b[1]) as [
-                  number,
-                  number,
-                ][];
-
-                for (let i = 1; i < points.length; ++i) {
-                  const pnt = points[i];
-                  pnt[1] += deltaY / (len - i);
-                }
+              const height = element.height + deltaY;
+              if (height <= 0) {
+                break;
               }
 
               mutateElement(element, {
-                height: element.height + deltaY,
-                points,
+                height,
+                points:
+                  element.points.length > 0
+                    ? rescalePoints(1, height, element.points)
+                    : undefined,
               });
+
               break;
             }
             case "e": {
-              let points;
-              if (element.points.length > 0) {
-                const len = element.points.length;
-                points = [...element.points].sort((a, b) => a[0] - b[0]) as [
-                  number,
-                  number,
-                ][];
-
-                for (let i = 1; i < points.length; ++i) {
-                  const pnt = points[i];
-                  pnt[0] += deltaX / (len - i);
-                }
+              const width = element.width + deltaX;
+              if (width <= 0) {
+                break;
               }
 
               mutateElement(element, {
-                width: element.width + deltaX,
-                points,
+                width,
+                points:
+                  element.points.length > 0
+                    ? rescalePoints(0, width, element.points)
+                    : undefined,
               });
               break;
             }
diff --git a/src/element/mutateElement.ts b/src/element/mutateElement.ts
index 58df4699..0b52a1f7 100644
--- a/src/element/mutateElement.ts
+++ b/src/element/mutateElement.ts
@@ -2,6 +2,7 @@ import { ExcalidrawElement } from "./types";
 import { randomSeed } from "roughjs/bin/math";
 import { invalidateShapeForElement } from "../renderer/renderElement";
 import { globalSceneState } from "../scene";
+import { getSizeFromPoints } from "../points";
 
 type ElementUpdate<TElement extends ExcalidrawElement> = Omit<
   Partial<TElement>,
@@ -18,6 +19,10 @@ export function mutateElement<TElement extends ExcalidrawElement>(
 ) {
   const mutableElement = element as any;
 
+  if (typeof updates.points !== "undefined") {
+    updates = { ...getSizeFromPoints(updates.points!), ...updates };
+  }
+
   for (const key in updates) {
     const value = (updates as any)[key];
     if (typeof value !== "undefined") {
diff --git a/src/points.ts b/src/points.ts
new file mode 100644
index 00000000..3b2ba501
--- /dev/null
+++ b/src/points.ts
@@ -0,0 +1,46 @@
+import { Point } from "./types";
+
+export function getSizeFromPoints(points: readonly Point[]) {
+  const xs = points.map(point => point[0]);
+  const ys = points.map(point => point[1]);
+  return {
+    width: Math.max(...xs) - Math.min(...xs),
+    height: Math.max(...ys) - Math.min(...ys),
+  };
+}
+export function rescalePoints(
+  dimension: 0 | 1,
+  nextDimensionSize: number,
+  prevPoints: readonly Point[],
+): readonly Point[] {
+  const prevDimValues = prevPoints.map(point => point[dimension]);
+  const prevMaxDimension = Math.max(...prevDimValues);
+  const prevMinDimension = Math.min(...prevDimValues);
+  const prevDimensionSize = prevMaxDimension - prevMinDimension;
+
+  const dimensionScaleFactor = nextDimensionSize / prevDimensionSize;
+
+  let nextMinDimension = Infinity;
+
+  const scaledPoints = prevPoints.map(prevPoint =>
+    prevPoint.map((value, currentDimension) => {
+      if (currentDimension !== dimension) {
+        return value;
+      }
+      const scaledValue = value * dimensionScaleFactor;
+      nextMinDimension = Math.min(scaledValue, nextMinDimension);
+      return scaledValue;
+    }),
+  );
+
+  const translation = prevMinDimension - nextMinDimension;
+
+  const nextPoints = scaledPoints.map(
+    scaledPoint =>
+      scaledPoint.map((value, currentDimension) => {
+        return currentDimension === dimension ? value + translation : value;
+      }) as [number, number],
+  );
+
+  return nextPoints;
+}