diff --git a/java/src/org/futo/inputmethod/keyboard/PointerTracker.java b/java/src/org/futo/inputmethod/keyboard/PointerTracker.java
index 92f891afdaedc95a8182cf96ebe44411a56de32c..57e09e32bffd41f3fd4b25d140aa39f8608007a6 100644
--- a/java/src/org/futo/inputmethod/keyboard/PointerTracker.java
+++ b/java/src/org/futo/inputmethod/keyboard/PointerTracker.java
@@ -704,7 +704,11 @@ public final class PointerTracker implements PointerTrackerQueue.Element,
             // {@link #setKeyboard}. In those cases, we should update key according to the new
             // keyboard layout.
             if (callListenerOnPressAndCheckKeyboardLayoutChange(key, 0 /* repeatCount */)) {
-                key = onDownKey(x, y, eventTime);
+                key = getKeyOn(x, y);
+                if(!key.isModifier())
+                    key = null;
+                else
+                    key = onDownKey(x, y, eventTime);
             }
 
             if(key == null) return;