diff --git a/src/yuzu/configuration/configure_input_player.cpp b/src/yuzu/configuration/configure_input_player.cpp
index 96dec50e2c..15ac30f12a 100644
--- a/src/yuzu/configuration/configure_input_player.cpp
+++ b/src/yuzu/configuration/configure_input_player.cpp
@@ -541,18 +541,19 @@ void ConfigureInputPlayer::HandleClick(
     button->setText(tr("[press key]"));
     button->setFocus();
 
-    const auto iter = std::find(button_map.begin(), button_map.end(), button);
-    ASSERT(iter != button_map.end());
-    const auto index = std::distance(button_map.begin(), iter);
-    ASSERT(index < Settings::NativeButton::NumButtons && index >= 0);
+    // Keyboard keys can only be used as button devices
+    want_keyboard_keys = type == InputCommon::Polling::DeviceType::Button;
+    if (want_keyboard_keys) {
+        const auto iter = std::find(button_map.begin(), button_map.end(), button);
+        ASSERT(iter != button_map.end());
+        const auto index = std::distance(button_map.begin(), iter);
+        ASSERT(index < Settings::NativeButton::NumButtons && index >= 0);
+    }
 
     input_setter = new_input_setter;
 
     device_pollers = InputCommon::Polling::GetPollers(type);
 
-    // Keyboard keys can only be used as button devices
-    want_keyboard_keys = type == InputCommon::Polling::DeviceType::Button;
-
     for (auto& poller : device_pollers) {
         poller->Start();
     }
diff --git a/src/yuzu/configuration/configure_mouse_advanced.cpp b/src/yuzu/configuration/configure_mouse_advanced.cpp
index 0a4abe34f3..e0647ea5bc 100644
--- a/src/yuzu/configuration/configure_mouse_advanced.cpp
+++ b/src/yuzu/configuration/configure_mouse_advanced.cpp
@@ -184,18 +184,19 @@ void ConfigureMouseAdvanced::HandleClick(
     button->setText(tr("[press key]"));
     button->setFocus();
 
-    const auto iter = std::find(button_map.begin(), button_map.end(), button);
-    ASSERT(iter != button_map.end());
-    const auto index = std::distance(button_map.begin(), iter);
-    ASSERT(index < Settings::NativeButton::NumButtons && index >= 0);
+    // Keyboard keys can only be used as button devices
+    want_keyboard_keys = type == InputCommon::Polling::DeviceType::Button;
+    if (want_keyboard_keys) {
+        const auto iter = std::find(button_map.begin(), button_map.end(), button);
+        ASSERT(iter != button_map.end());
+        const auto index = std::distance(button_map.begin(), iter);
+        ASSERT(index < Settings::NativeButton::NumButtons && index >= 0);
+    }
 
     input_setter = new_input_setter;
 
     device_pollers = InputCommon::Polling::GetPollers(type);
 
-    // Keyboard keys can only be used as button devices
-    want_keyboard_keys = type == InputCommon::Polling::DeviceType::Button;
-
     for (auto& poller : device_pollers) {
         poller->Start();
     }