XIAO SAMD21G with Arduino IDE, INPUT_PULLUP not working

Try this.

//#define BUTTON   D3   // aka PA11 and EXTINT[11]
#define BUTTON   A3   // aka PA11 and EXTINT[11]

I have tried this.

#define PORT A3
void setup() {
  Serial.begin(115200);

  pinMode(LED_BUILTIN, OUTPUT);     // Yellow LED
  digitalWrite(LED_BUILTIN, HIGH);  // HIGH:OFF, LOW:ON

  pinMode(PORT, INPUT_PULLUP);

}
// Connect between PORT and GND to a switch.
void loop() {
  digitalWrite(LED_BUILTIN, digitalRead(PORT));
  Serial.println(digitalRead(PORT));
  delay(100);
}