Test for button press every 50ms. That's more than enough.

src/controller.cpp

@@ -6,7 +6,10 @@ void Controller::handle() {
 		handle_rfid();
 		last_rfid_check = millis();
 	}
-	handle_buttons();
+	if (last_button_check + 10 < millis() || last_button_check > millis()) {
+		handle_buttons();
+		last_button_check = millis();
+	}
 }
 
 void Controller::handle_buttons() {