BLE reconnect after light sleep takes too long (XIAO ESP32C3)

static const uint8_t _hidReportDescriptor[]
    = {
          USAGE_PAGE(1), 0x0C,                  // USAGE_PAGE (Consumer)
          USAGE(1), 0x01,                       // USAGE (Consumer Control)
          COLLECTION(1), 0x01,                  // COLLECTION (Application)
          REPORT_ID(1), BLE_CONSUMER_REPORT_ID, //   REPORT_ID (1)
          LOGICAL_MINIMUM(1), 0x00,             //   LOGICAL_MINIMUM (0)
          LOGICAL_MAXIMUM(1), 0x01,             //   LOGICAL_MAXIMUM (1)
          REPORT_SIZE(1), 0x01,                 //   REPORT_SIZE (1)
          REPORT_COUNT(1), 0x0C,                //   REPORT_COUNT (12)
          USAGE(1), 0x30,                       //   Power
          USAGE(1), 0x41,                       //   Menu Pick (Select)
          USAGE(1), 0x46,                       //   Menu Escape (Back)
          USAGE(2), 0x23, 0x02,                 //   Home
          USAGE(1), 0xE9,                       //   Volume Increase
          USAGE(1), 0xEA,                       //   Volume Decrease
          USAGE(1), 0xE2,                       //   Volume Mute
          USAGE(1), 0x40,                       //   Menu
          USAGE(1), 0x42,                       //   Menu Up
          USAGE(1), 0x43,                       //   Menu Down
          USAGE(1), 0x44,                       //   Menu Left
          USAGE(1), 0x45,                       //   Menu Right
          HIDINPUT(1), 0x02,                    //   INPUT (Data,Var,Abs,No Wrap,Linear,Preferred State,No Null Position)
          REPORT_SIZE(1), 0x04,                 //   REPORT_SIZE (4) - Padding 4 bit
          REPORT_COUNT(1), 0x01,                //   REPORT_COUNT (1)
          HIDINPUT(1), 0x03,                    //   INPUT
          END_COLLECTION(0)                     // END COLLECTION
      };

void BLERemote::taskServer(void* pvParameter) {
    BLERemote* BLERemoteInstance = (BLERemote*)pvParameter;

    NimBLEDevice::init(BLERemoteInstance->deviceName);
    NimBLEDevice::setSecurityAuth(true, false, false);

    NimBLEServer* pServer = NimBLEDevice::createServer();
    pServer->setCallbacks(BLERemoteInstance->connectionStatus);
    BLERemoteInstance->hid           = new NimBLEHIDDevice(pServer);
    BLERemoteInstance->inputConsumer = BLERemoteInstance->hid->inputReport(BLE_CONSUMER_REPORT_ID);
    BLERemoteInstance->hid->manufacturer()->setValue(BLERemoteInstance->deviceManufacturer);
    BLERemoteInstance->hid->pnp(BLE_VEND_SRC, BLE_VEND_ID, BLE_PROD_ID, BLE_PROD_VER);
    BLERemoteInstance->hid->hidInfo(0x00, 0x01);
    BLERemoteInstance->hid->setBatteryLevel(BLERemoteInstance->batteryLevel);
    BLERemoteInstance->hid->reportMap((uint8_t*)_hidReportDescriptor, sizeof(_hidReportDescriptor));
    BLERemoteInstance->hid->startServices();
    // BLERemoteInstance->onStarted(pServer);

    NimBLEAdvertising* pAdvertising = pServer->getAdvertising();
    pAdvertising->setAppearance(BLE_HID_REMOTE_APPEARANCE);
    pAdvertising->addServiceUUID(BLERemoteInstance->hid->hidService()->getUUID());
    pAdvertising->start();

    vTaskDelay(portMAX_DELAY);
}

Only relevent parts of the code is above. Looks like I’m uaing the default parameters. Which parameters should I start experimenting with?