diff --git a/packages/excalidraw/components/App.tsx b/packages/excalidraw/components/App.tsx
index 5122151ed..8a5184782 100644
--- a/packages/excalidraw/components/App.tsx
+++ b/packages/excalidraw/components/App.tsx
@@ -6289,8 +6289,6 @@ class App extends React.Component<AppProps, AppState> {
   ) => {
     this.maybeCleanupAfterMissingPointerUp(event.nativeEvent);
 
-    this.lastPointerDownEvent = event;
-
     this.maybeUnfollowRemoteUser();
 
     if (this.state.searchMatches) {
@@ -6390,6 +6388,8 @@ class App extends React.Component<AppProps, AppState> {
       return;
     }
 
+    this.lastPointerDownEvent = event;
+
     // we must exit before we set `cursorButton` state and `savePointer`
     // else it will send pointer state & laser pointer events in collab when
     // panning