diff --git a/src/index.tsx b/src/index.tsx
index 82db6097c7..e5156e1f05 100644
--- a/src/index.tsx
+++ b/src/index.tsx
@@ -15,15 +15,94 @@ type ExcaliburTextElement = ExcaliburElement & {
 
 var elements = Array.of<ExcaliburElement>();
 
-function isInsideAnElement(x: number, y: number) {
-  return (element: ExcaliburElement) => {
+// https://stackoverflow.com/a/6853926/232122
+function distanceBetweenPointAndSegment(
+  x: number,
+  y: number,
+  x1: number,
+  y1: number,
+  x2: number,
+  y2: number
+) {
+  const A = x - x1;
+  const B = y - y1;
+  const C = x2 - x1;
+  const D = y2 - y1;
+
+  const dot = A * C + B * D;
+  const lenSquare = C * C + D * D;
+  let param = -1;
+  if (lenSquare !== 0) {
+    // in case of 0 length line
+    param = dot / lenSquare;
+  }
+
+  let xx, yy;
+  if (param < 0) {
+    xx = x1;
+    yy = y1;
+  } else if (param > 1) {
+    xx = x2;
+    yy = y2;
+  } else {
+    xx = x1 + param * C;
+    yy = y1 + param * D;
+  }
+
+  const dx = x - xx;
+  const dy = y - yy;
+  return Math.sqrt(dx * dx + dy * dy);
+}
+
+function hitTest(element: ExcaliburElement, x: number, y: number): boolean {
+  // For shapes that are composed of lines, we only enable point-selection when the distance
+  // of the click is less than x pixels of any of the lines that the shape is composed of
+  const lineThreshold = 10;
+
+  if (
+    element.type === "rectangle" ||
+    // There doesn't seem to be a closed form solution for the distance between
+    // a point and an ellipse, let's assume it's a rectangle for now...
+    element.type === "ellipse"
+  ) {
+    const x1 = getElementAbsoluteX1(element);
+    const x2 = getElementAbsoluteX2(element);
+    const y1 = getElementAbsoluteY1(element);
+    const y2 = getElementAbsoluteY2(element);
+
+    // (x1, y1) --A-- (x2, y1)
+    //    |D             |B
+    // (x1, y2) --C-- (x2, y2)
+    return (
+      distanceBetweenPointAndSegment(x, y, x1, y1, x2, y1) < lineThreshold || // A
+      distanceBetweenPointAndSegment(x, y, x2, y1, x2, y2) < lineThreshold || // B
+      distanceBetweenPointAndSegment(x, y, x2, y2, x1, y2) < lineThreshold || // C
+      distanceBetweenPointAndSegment(x, y, x1, y2, x1, y1) < lineThreshold // D
+    );
+  } else if (element.type === "arrow") {
+    let [x1, y1, x2, y2, x3, y3, x4, y4] = getArrowPoints(element);
+    // The computation is done at the origin, we need to add a translation
+    x -= element.x;
+    y -= element.y;
+
+    return (
+      //    \
+      distanceBetweenPointAndSegment(x, y, x3, y3, x2, y2) < lineThreshold ||
+      // -----
+      distanceBetweenPointAndSegment(x, y, x1, y1, x2, y2) < lineThreshold ||
+      //    /
+      distanceBetweenPointAndSegment(x, y, x4, y4, x2, y2) < lineThreshold
+    );
+  } else if (element.type === "text") {
     const x1 = getElementAbsoluteX1(element);
     const x2 = getElementAbsoluteX2(element);
     const y1 = getElementAbsoluteY1(element);
     const y2 = getElementAbsoluteY2(element);
 
     return x >= x1 && x <= x2 && y >= y1 && y <= y2;
-  };
+  } else {
+    throw new Error("Unimplemented type " + element.type);
+  }
 }
 
 function newElement(type: string, x: number, y: number, width = 0, height = 0) {
@@ -139,6 +218,26 @@ function isTextElement(
   return element.type === "text";
 }
 
+function getArrowPoints(element: ExcaliburElement) {
+  const x1 = 0;
+  const y1 = 0;
+  const x2 = element.width;
+  const y2 = element.height;
+
+  const size = 30; // pixels
+  const distance = Math.sqrt(Math.pow(x2 - x1, 2) + Math.pow(y2 - y1, 2));
+  // Scale down the arrow until we hit a certain size so that it doesn't look weird
+  const minSize = Math.min(size, distance / 2);
+  const xs = x2 - ((x2 - x1) / distance) * minSize;
+  const ys = y2 - ((y2 - y1) / distance) * minSize;
+
+  const angle = 20; // degrees
+  const [x3, y3] = rotate(xs, ys, x2, y2, (-angle * Math.PI) / 180);
+  const [x4, y4] = rotate(xs, ys, x2, y2, (angle * Math.PI) / 180);
+
+  return [x1, y1, x2, y2, x3, y3, x4, y4];
+}
+
 function generateDraw(element: ExcaliburElement) {
   if (element.type === "selection") {
     element.draw = (rc, context) => {
@@ -167,22 +266,7 @@ function generateDraw(element: ExcaliburElement) {
       context.translate(-element.x, -element.y);
     };
   } else if (element.type === "arrow") {
-    const x1 = 0;
-    const y1 = 0;
-    const x2 = element.width;
-    const y2 = element.height;
-
-    const size = 30; // pixels
-    const distance = Math.sqrt(Math.pow(x2 - x1, 2) + Math.pow(y2 - y1, 2));
-    // Scale down the arrow until we hit a certain size so that it doesn't look weird
-    const minSize = Math.min(size, distance / 2);
-    const xs = x2 - ((x2 - x1) / distance) * minSize;
-    const ys = y2 - ((y2 - y1) / distance) * minSize;
-
-    const angle = 20; // degrees
-    const [x3, y3] = rotate(xs, ys, x2, y2, (-angle * Math.PI) / 180);
-    const [x4, y4] = rotate(xs, ys, x2, y2, (angle * Math.PI) / 180);
-
+    const [x1, y1, x2, y2, x3, y3, x4, y4] = getArrowPoints(element);
     const shapes = [
       //    \
       generator.line(x3, y3, x2, y2),
@@ -443,7 +527,7 @@ class App extends React.Component<{}, AppState> {
               const cursorStyle = document.documentElement.style.cursor;
               if (this.state.elementType === "selection") {
                 const selectedElement = elements.find(element => {
-                  const isSelected = isInsideAnElement(x, y)(element);
+                  const isSelected = hitTest(element, x, y);
                   if (isSelected) {
                     element.isSelected = true;
                   }