Test for button press every 50ms. That's more than enough.

include/controller.h

@@ -14,6 +14,7 @@ class Controller {
 	Playlist current_playlist;
 	bool is_rfid_present = false;
 	unsigned long last_rfid_check = 0;
+	unsigned long last_button_check = 0;
 	uint8_t button_pressed = 0;
 	unsigned long button_pressed_since = 0;
 	bool button_already_processed = false;

src/controller.cpp

@@ -6,7 +6,10 @@ void Controller::handle() {
 		handle_rfid();
 		last_rfid_check = millis();
 	}
+	if (last_button_check + 10 < millis() || last_button_check > millis()) {
 		handle_buttons();
+		last_button_check = millis();
+	}
 }
 
 void Controller::handle_buttons() {