diff --git a/sys/dev/mmc/bridge.h b/sys/dev/mmc/bridge.h index 502f433c0d5..53e61b48f32 100644 --- a/sys/dev/mmc/bridge.h +++ b/sys/dev/mmc/bridge.h @@ -137,6 +137,10 @@ enum mmc_card_mode { mode_mmc, mode_sd }; +enum mmc_retune_req { + retune_req_none = 0, retune_req_normal, retune_req_reset +}; + struct mmc_host { int f_min; int f_max; @@ -178,7 +182,7 @@ struct mmc_host { extern driver_t mmc_driver; extern devclass_t mmc_devclass; -#define MMC_VERSION 3 +#define MMC_VERSION 4 #define MMC_DECLARE_BRIDGE(name) \ DRIVER_MODULE(mmc, name, mmc_driver, mmc_devclass, NULL, NULL); \ diff --git a/sys/dev/mmc/mmc.c b/sys/dev/mmc/mmc.c index af34bd76acb..69a449dea58 100644 --- a/sys/dev/mmc/mmc.c +++ b/sys/dev/mmc/mmc.c @@ -88,14 +88,14 @@ struct mmc_ivars { uint8_t raw_ext_csd[MMC_EXTCSD_SIZE]; /* Raw bits of the EXT_CSD */ uint32_t raw_sd_status[16]; /* Raw bits of the SD_STATUS */ uint16_t rca; + u_char read_only; /* True when the device is read-only */ + u_char high_cap; /* High Capacity device (block addressed) */ enum mmc_card_mode mode; + enum mmc_bus_width bus_width; /* Bus width to use */ struct mmc_cid cid; /* cid decoded */ struct mmc_csd csd; /* csd decoded */ struct mmc_scr scr; /* scr decoded */ struct mmc_sd_status sd_status; /* SD_STATUS decoded */ - u_char read_only; /* True when the device is read-only */ - u_char bus_width; /* Bus width to use */ - u_char high_cap; /* High Capacity card (block addressed) */ uint32_t sec_count; /* Card capacity in 512byte blocks */ uint32_t timings; /* Mask of bus timings supported */ uint32_t vccq_120; /* Mask of bus timings at VCCQ of 1.2 V */ @@ -127,8 +127,10 @@ static int mmc_read_ivar(device_t bus, device_t child, int which, uintptr_t *result); static int mmc_release_bus(device_t busdev, device_t dev); static int mmc_resume(device_t dev); +static void mmc_retune_pause(device_t busdev, device_t dev, bool retune); +static void mmc_retune_unpause(device_t busdev, device_t dev); static int mmc_suspend(device_t dev); -static int mmc_wait_for_request(device_t brdev, device_t reqdev, +static int mmc_wait_for_request(device_t busdev, device_t dev, struct mmc_request *req); static int mmc_write_ivar(device_t bus, device_t child, int which, uintptr_t value); @@ -155,21 +157,23 @@ static void mmc_decode_cid_mmc(uint32_t *raw_cid, struct mmc_cid *cid, bool is_4_41p); static void mmc_decode_cid_sd(uint32_t *raw_cid, struct mmc_cid *cid); static void mmc_decode_csd_mmc(uint32_t *raw_csd, struct mmc_csd *csd); -static void mmc_decode_csd_sd(uint32_t *raw_csd, struct mmc_csd *csd); +static int mmc_decode_csd_sd(uint32_t *raw_csd, struct mmc_csd *csd); static void mmc_delayed_attach(void *xsc); -static int mmc_delete_cards(struct mmc_softc *sc); +static int mmc_delete_cards(struct mmc_softc *sc, bool final); static void mmc_discover_cards(struct mmc_softc *sc); static void mmc_format_card_id_string(struct mmc_ivars *ivar); static void mmc_go_discovery(struct mmc_softc *sc); static uint32_t mmc_get_bits(uint32_t *bits, int bit_len, int start, int size); static int mmc_highest_voltage(uint32_t ocr); +static bool mmc_host_timing(device_t dev, enum mmc_bus_timing timing); static void mmc_idle_cards(struct mmc_softc *sc); static void mmc_ms_delay(int ms); static void mmc_log_card(device_t dev, struct mmc_ivars *ivar, int newcard); static void mmc_power_down(struct mmc_softc *sc); static void mmc_power_up(struct mmc_softc *sc); static void mmc_rescan_cards(struct mmc_softc *sc); +static int mmc_retune(device_t busdev, device_t dev, bool reset); static void mmc_scan(struct mmc_softc *sc); static int mmc_sd_switch(struct mmc_softc *sc, uint8_t mode, uint8_t grp, uint8_t value, uint8_t *res); @@ -183,15 +187,23 @@ static int mmc_send_op_cond(struct mmc_softc *sc, uint32_t ocr, uint32_t *rocr); static int mmc_send_relative_addr(struct mmc_softc *sc, uint32_t *resp); static int mmc_set_blocklen(struct mmc_softc *sc, uint32_t len); -static int mmc_set_card_bus_width(struct mmc_softc *sc, struct mmc_ivars *ivar); +static int mmc_set_card_bus_width(struct mmc_softc *sc, struct mmc_ivars *ivar, + enum mmc_bus_timing timing); static int mmc_set_power_class(struct mmc_softc *sc, struct mmc_ivars *ivar); static int mmc_set_relative_addr(struct mmc_softc *sc, uint16_t resp); static int mmc_set_timing(struct mmc_softc *sc, struct mmc_ivars *ivar, enum mmc_bus_timing timing); +static int mmc_set_vccq(struct mmc_softc *sc, struct mmc_ivars *ivar, + enum mmc_bus_timing timing); +static int mmc_switch_to_hs200(struct mmc_softc *sc, struct mmc_ivars *ivar, + uint32_t clock); +static int mmc_switch_to_hs400(struct mmc_softc *sc, struct mmc_ivars *ivar, + uint32_t max_dtr, enum mmc_bus_timing max_timing); static int mmc_test_bus_width(struct mmc_softc *sc); static uint32_t mmc_timing_to_dtr(struct mmc_ivars *ivar, enum mmc_bus_timing timing); static const char *mmc_timing_to_string(enum mmc_bus_timing timing); +static void mmc_update_child_list(struct mmc_softc *sc); static int mmc_wait_for_command(struct mmc_softc *sc, uint32_t opcode, uint32_t arg, uint32_t flags, uint32_t *resp, int retries); static int mmc_wait_for_req(struct mmc_softc *sc, struct mmc_request *req); @@ -235,7 +247,8 @@ mmc_detach(device_t dev) struct mmc_softc *sc = device_get_softc(dev); int err; - if ((err = mmc_delete_cards(sc)) != 0) + err = mmc_delete_cards(sc, true); + if (err != 0) return (err); mmc_power_down(sc); MMC_LOCK_DESTROY(sc); @@ -250,10 +263,21 @@ mmc_suspend(device_t dev) int err; err = bus_generic_suspend(dev); - if (err) + if (err != 0) + return (err); + /* + * We power down with the bus acquired here, mainly so that no device + * is selected any longer and sc->last_rca gets set to 0. Otherwise, + * the deselect as part of the bus acquisition in mmc_scan() may fail + * during resume, as the bus isn't powered up again before later in + * mmc_go_discovery(). + */ + err = mmc_acquire_bus(dev, dev); + if (err != 0) return (err); mmc_power_down(sc); - return (0); + err = mmc_release_bus(dev, dev); + return (err); } static int @@ -270,7 +294,8 @@ mmc_acquire_bus(device_t busdev, device_t dev) { struct mmc_softc *sc; struct mmc_ivars *ivar; - int err, rca; + int err; + uint16_t rca; enum mmc_bus_timing timing; err = MMCBR_ACQUIRE_HOST(device_get_parent(busdev), busdev); @@ -294,12 +319,27 @@ mmc_acquire_bus(device_t busdev, device_t dev) rca = ivar->rca; if (sc->last_rca != rca) { if (mmc_select_card(sc, rca) != MMC_ERR_NONE) { - device_printf(sc->dev, "Card at relative " - "address %d failed to select.\n", rca); + device_printf(busdev, "Card at relative " + "address %d failed to select\n", rca); return (ENXIO); } sc->last_rca = rca; timing = mmcbr_get_timing(busdev); + /* + * For eMMC modes, setting/updating bus width and VCCQ + * only really is necessary if there actually is more + * than one device on the bus as generally that already + * had to be done by mmc_calculate_clock() or one of + * its calees. Moreover, setting the bus width anew + * can trigger re-tuning (via a CRC error on the next + * CMD), even if not switching between devices an the + * previously selected one is still tuned. Obviously, + * we need to re-tune the host controller if devices + * are actually switched, though. + */ + if (timing >= bus_timing_mmc_ddr52 && + sc->child_count == 1) + return (0); /* Prepare bus width for the new card. */ if (bootverbose || mmc_debug) { device_printf(busdev, @@ -308,38 +348,34 @@ mmc_acquire_bus(device_t busdev, device_t dev) (ivar->bus_width == bus_width_8) ? 8 : 1, mmc_timing_to_string(timing)); } - if (mmc_set_card_bus_width(sc, ivar) != MMC_ERR_NONE) { - device_printf(sc->dev, "Card at relative " - "address %d failed to set bus width.\n", - rca); - return (ENXIO); - } - if (isset(&ivar->vccq_120, timing)) - mmcbr_set_vccq(busdev, vccq_120); - else if (isset(&ivar->vccq_180, timing)) - mmcbr_set_vccq(busdev, vccq_180); - else - mmcbr_set_vccq(busdev, vccq_330); - if (mmcbr_switch_vccq(busdev) != 0) { - device_printf(sc->dev, "Failed to set VCCQ " - "for card at relative address %d.\n", rca); - return (ENXIO); - } - if (mmc_set_power_class(sc, ivar) != MMC_ERR_NONE) { - device_printf(sc->dev, "Card at relative " - "address %d failed to set power class.\n", + if (mmc_set_card_bus_width(sc, ivar, timing) != + MMC_ERR_NONE) { + device_printf(busdev, "Card at relative " + "address %d failed to set bus width\n", rca); return (ENXIO); } mmcbr_set_bus_width(busdev, ivar->bus_width); mmcbr_update_ios(busdev); + if (mmc_set_vccq(sc, ivar, timing) != MMC_ERR_NONE) { + device_printf(busdev, "Failed to set VCCQ " + "for card at relative address %d\n", rca); + return (ENXIO); + } + if (timing >= bus_timing_mmc_hs200 && + mmc_retune(busdev, dev, true) != 0) { + device_printf(busdev, "Card at relative " + "address %d failed to re-tune\n", rca); + return (ENXIO); + } } } else { /* * If there's a card selected, stand down. */ if (sc->last_rca != 0) { - mmc_select_card(sc, 0); + if (mmc_select_card(sc, 0) != MMC_ERR_NONE) + return (ENXIO); sc->last_rca = 0; } } @@ -407,7 +443,7 @@ mmc_wait_for_req(struct mmc_softc *sc, struct mmc_request *req) req->done = mmc_wakeup; req->done_data = sc; - if (mmc_debug > 1) { + if (__predict_false(mmc_debug > 1)) { device_printf(sc->dev, "REQUEST: CMD%d arg %#x flags %#x", req->cmd->opcode, req->cmd->arg, req->cmd->flags); if (req->cmd->data) { @@ -420,18 +456,66 @@ mmc_wait_for_req(struct mmc_softc *sc, struct mmc_request *req) while ((req->flags & MMC_REQ_DONE) == 0) msleep(req, &sc->sc_mtx, 0, "mmcreq", 0); MMC_UNLOCK(sc); - if (mmc_debug > 2 || (mmc_debug > 0 && req->cmd->error != MMC_ERR_NONE)) + if (__predict_false(mmc_debug > 2 || (mmc_debug > 0 && + req->cmd->error != MMC_ERR_NONE))) device_printf(sc->dev, "CMD%d RESULT: %d\n", req->cmd->opcode, req->cmd->error); return (0); } static int -mmc_wait_for_request(device_t brdev, device_t reqdev __unused, - struct mmc_request *req) +mmc_wait_for_request(device_t busdev, device_t dev, struct mmc_request *req) { - struct mmc_softc *sc = device_get_softc(brdev); + struct mmc_softc *sc; + struct mmc_ivars *ivar; + int err, i; + enum mmc_retune_req retune_req; + sc = device_get_softc(busdev); + KASSERT(sc->owner != NULL, + ("%s: Request from %s without bus being acquired.", __func__, + device_get_nameunit(dev))); + + /* + * Unless no device is selected or re-tuning is already ongoing, + * execute re-tuning if a) the bridge is requesting to do so and + * re-tuning hasn't been otherwise paused, or b) if a child asked + * to be re-tuned prior to pausing (see also mmc_retune_pause()). + */ + if (__predict_false(sc->last_rca != 0 && sc->retune_ongoing == 0 && + (((retune_req = mmcbr_get_retune_req(busdev)) != retune_req_none && + sc->retune_paused == 0) || sc->retune_needed == 1))) { + if (__predict_false(mmc_debug > 1)) { + device_printf(busdev, + "Re-tuning with%s circuit reset required\n", + retune_req == retune_req_reset ? "" : "out"); + } + if (device_get_parent(dev) == busdev) + ivar = device_get_ivars(dev); + else { + for (i = 0; i < sc->child_count; i++) { + ivar = device_get_ivars(sc->child_list[i]); + if (ivar->rca == sc->last_rca) + break; + } + if (ivar->rca != sc->last_rca) + return (EINVAL); + } + sc->retune_ongoing = 1; + err = mmc_retune(busdev, dev, retune_req == retune_req_reset); + sc->retune_ongoing = 0; + switch (err) { + case MMC_ERR_NONE: + case MMC_ERR_FAILED: /* Re-tune error but still might work */ + break; + case MMC_ERR_BADCRC: /* Switch failure on HS400 recovery */ + return (ENXIO); + case MMC_ERR_INVALID: /* Driver implementation b0rken */ + default: /* Unknown error, should not happen */ + return (EINVAL); + } + sc->retune_needed = 0; + } return (mmc_wait_for_req(sc, req)); } @@ -599,11 +683,14 @@ mmc_power_down(struct mmc_softc *sc) static int mmc_select_card(struct mmc_softc *sc, uint16_t rca) { - int flags; + int err, flags; flags = (rca ? MMC_RSP_R1B : MMC_RSP_NONE) | MMC_CMD_AC; - return (mmc_wait_for_command(sc, MMC_SELECT_CARD, (uint32_t)rca << 16, - flags, NULL, CMD_RETRIES)); + sc->retune_paused++; + err = mmc_wait_for_command(sc, MMC_SELECT_CARD, (uint32_t)rca << 16, + flags, NULL, CMD_RETRIES); + sc->retune_paused--; + return (err); } static int @@ -635,7 +722,8 @@ mmc_sd_switch(struct mmc_softc *sc, uint8_t mode, uint8_t grp, uint8_t value, } static int -mmc_set_card_bus_width(struct mmc_softc *sc, struct mmc_ivars *ivar) +mmc_set_card_bus_width(struct mmc_softc *sc, struct mmc_ivars *ivar, + enum mmc_bus_timing timing) { struct mmc_command cmd; int err; @@ -668,28 +756,33 @@ mmc_set_card_bus_width(struct mmc_softc *sc, struct mmc_ivars *ivar) } else { switch (ivar->bus_width) { case bus_width_1: + if (timing == bus_timing_mmc_hs400 || + timing == bus_timing_mmc_hs400es) + return (MMC_ERR_INVALID); value = EXT_CSD_BUS_WIDTH_1; break; case bus_width_4: - switch (mmcbr_get_timing(sc->dev)) { + switch (timing) { case bus_timing_mmc_ddr52: - case bus_timing_mmc_hs200: - case bus_timing_mmc_hs400: - case bus_timing_mmc_hs400es: value = EXT_CSD_BUS_WIDTH_4_DDR; break; + case bus_timing_mmc_hs400: + case bus_timing_mmc_hs400es: + return (MMC_ERR_INVALID); default: value = EXT_CSD_BUS_WIDTH_4; break; } break; case bus_width_8: - switch (mmcbr_get_timing(sc->dev)) { - case bus_timing_mmc_ddr52: - case bus_timing_mmc_hs200: - case bus_timing_mmc_hs400: + value = 0; + switch (timing) { case bus_timing_mmc_hs400es: - value = EXT_CSD_BUS_WIDTH_8_DDR; + value = EXT_CSD_BUS_WIDTH_ES; + /* FALLTHROUGH */ + case bus_timing_mmc_ddr52: + case bus_timing_mmc_hs400: + value |= EXT_CSD_BUS_WIDTH_8_DDR; break; default: value = EXT_CSD_BUS_WIDTH_8; @@ -814,6 +907,13 @@ mmc_set_timing(struct mmc_softc *sc, struct mmc_ivars *ivar, case bus_timing_mmc_ddr52: value = EXT_CSD_HS_TIMING_HS; break; + case bus_timing_mmc_hs200: + value = EXT_CSD_HS_TIMING_HS200; + break; + case bus_timing_mmc_hs400: + case bus_timing_mmc_hs400es: + value = EXT_CSD_HS_TIMING_HS400; + break; default: return (MMC_ERR_INVALID); } @@ -830,6 +930,23 @@ mmc_set_timing(struct mmc_softc *sc, struct mmc_ivars *ivar, return (err); } +static int +mmc_set_vccq(struct mmc_softc *sc, struct mmc_ivars *ivar, + enum mmc_bus_timing timing) +{ + + if (isset(&ivar->vccq_120, timing)) + mmcbr_set_vccq(sc->dev, vccq_120); + else if (isset(&ivar->vccq_180, timing)) + mmcbr_set_vccq(sc->dev, vccq_180); + else + mmcbr_set_vccq(sc->dev, vccq_330); + if (mmcbr_switch_vccq(sc->dev) != 0) + return (MMC_ERR_INVALID); + else + return (MMC_ERR_NONE); +} + static const uint8_t p8[8] = { 0x55, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; @@ -1037,7 +1154,7 @@ static const int cur_max[8] = { 1000, 5000, 10000, 25000, 35000, 45000, 800000, 200000 }; -static void +static int mmc_decode_csd_sd(uint32_t *raw_csd, struct mmc_csd *csd) { int v; @@ -1078,6 +1195,7 @@ mmc_decode_csd_sd(uint32_t *raw_csd, struct mmc_csd *csd) csd->r2w_factor = 1 << mmc_get_bits(raw_csd, 128, 26, 3); csd->write_bl_len = 1 << mmc_get_bits(raw_csd, 128, 22, 4); csd->write_bl_partial = mmc_get_bits(raw_csd, 128, 21, 1); + return (MMC_ERR_NONE); } else if (v == 1) { m = mmc_get_bits(raw_csd, 128, 115, 4); e = mmc_get_bits(raw_csd, 128, 112, 3); @@ -1101,8 +1219,9 @@ mmc_decode_csd_sd(uint32_t *raw_csd, struct mmc_csd *csd) csd->r2w_factor = 1 << mmc_get_bits(raw_csd, 128, 26, 3); csd->write_bl_len = 1 << mmc_get_bits(raw_csd, 128, 22, 4); csd->write_bl_partial = mmc_get_bits(raw_csd, 128, 21, 1); - } else - panic("unknown SD CSD version"); + return (MMC_ERR_NONE); + } + return (MMC_ERR_INVALID); } static void @@ -1366,6 +1485,53 @@ mmc_timing_to_string(enum mmc_bus_timing timing) return (""); } +static bool +mmc_host_timing(device_t dev, enum mmc_bus_timing timing) +{ + int host_caps; + + host_caps = mmcbr_get_caps(dev); + +#define HOST_TIMING_CAP(host_caps, cap) ({ \ + bool retval; \ + if (((host_caps) & (cap)) == (cap)) \ + retval = true; \ + else \ + retval = false; \ + retval; \ +}) + + switch (timing) { + case bus_timing_normal: + return (true); + case bus_timing_hs: + return (HOST_TIMING_CAP(host_caps, MMC_CAP_HSPEED)); + case bus_timing_uhs_sdr12: + return (HOST_TIMING_CAP(host_caps, MMC_CAP_UHS_SDR12)); + case bus_timing_uhs_sdr25: + return (HOST_TIMING_CAP(host_caps, MMC_CAP_UHS_SDR25)); + case bus_timing_uhs_ddr50: + return (HOST_TIMING_CAP(host_caps, MMC_CAP_UHS_DDR50)); + case bus_timing_uhs_sdr50: + return (HOST_TIMING_CAP(host_caps, MMC_CAP_UHS_SDR50)); + case bus_timing_uhs_sdr104: + return (HOST_TIMING_CAP(host_caps, MMC_CAP_UHS_SDR104)); + case bus_timing_mmc_ddr52: + return (HOST_TIMING_CAP(host_caps, MMC_CAP_MMC_DDR52)); + case bus_timing_mmc_hs200: + return (HOST_TIMING_CAP(host_caps, MMC_CAP_MMC_HS200)); + case bus_timing_mmc_hs400: + return (HOST_TIMING_CAP(host_caps, MMC_CAP_MMC_HS400)); + case bus_timing_mmc_hs400es: + return (HOST_TIMING_CAP(host_caps, MMC_CAP_MMC_HS400 | + MMC_CAP_MMC_ENH_STROBE)); + } + +#undef HOST_TIMING_CAP + + return (false); +} + static void mmc_log_card(device_t dev, struct mmc_ivars *ivar, int newcard) { @@ -1397,9 +1563,8 @@ mmc_discover_cards(struct mmc_softc *sc) u_char switch_res[64]; uint32_t raw_cid[4]; struct mmc_ivars *ivar = NULL; - device_t *devlist; device_t child; - int devcount, err, host_caps, i, newcard; + int err, host_caps, i, newcard; uint32_t resp, sec_count, status; uint16_t rca = 2; @@ -1407,6 +1572,7 @@ mmc_discover_cards(struct mmc_softc *sc) if (bootverbose || mmc_debug) device_printf(sc->dev, "Probing cards\n"); while (1) { + child = NULL; sc->squelched++; /* Errors are expected, squelch reporting. */ err = mmc_all_send_cid(sc, raw_cid); sc->squelched--; @@ -1417,18 +1583,14 @@ mmc_discover_cards(struct mmc_softc *sc) break; } newcard = 1; - if ((err = device_get_children(sc->dev, &devlist, - &devcount)) != 0) - return; - for (i = 0; i < devcount; i++) { - ivar = device_get_ivars(devlist[i]); + for (i = 0; i < sc->child_count; i++) { + ivar = device_get_ivars(sc->child_list[i]); if (memcmp(ivar->raw_cid, raw_cid, sizeof(raw_cid)) == 0) { newcard = 0; break; } } - free(devlist, M_TEMP); if (bootverbose || mmc_debug) { device_printf(sc->dev, "%sard detected (CID %08x%08x%08x%08x)\n", @@ -1451,7 +1613,7 @@ mmc_discover_cards(struct mmc_softc *sc) if (err != MMC_ERR_NONE) { device_printf(sc->dev, "Error getting RCA %d\n", err); - break; + goto free_ivar; } ivar->rca = resp >> 16; /* Get card CSD. */ @@ -1459,7 +1621,7 @@ mmc_discover_cards(struct mmc_softc *sc) if (err != MMC_ERR_NONE) { device_printf(sc->dev, "Error getting CSD %d\n", err); - break; + goto free_ivar; } if (bootverbose || mmc_debug) device_printf(sc->dev, @@ -1467,7 +1629,11 @@ mmc_discover_cards(struct mmc_softc *sc) newcard ? "New c" : "C", ivar->raw_csd[0], ivar->raw_csd[1], ivar->raw_csd[2], ivar->raw_csd[3]); - mmc_decode_csd_sd(ivar->raw_csd, &ivar->csd); + err = mmc_decode_csd_sd(ivar->raw_csd, &ivar->csd); + if (err != MMC_ERR_NONE) { + device_printf(sc->dev, "Error decoding CSD\n"); + goto free_ivar; + } ivar->sec_count = ivar->csd.capacity / MMC_SECTOR_SIZE; if (ivar->csd.csd_structure > 0) ivar->high_cap = 1; @@ -1480,12 +1646,12 @@ mmc_discover_cards(struct mmc_softc *sc) if (err != MMC_ERR_NONE) { device_printf(sc->dev, "Error reading card status %d\n", err); - break; + goto free_ivar; } if ((status & R1_CARD_IS_LOCKED) != 0) { device_printf(sc->dev, - "Card is password protected, skipping.\n"); - break; + "Card is password protected, skipping\n"); + goto free_ivar; } /* Get card SCR. Card must be selected to fetch it. */ @@ -1493,13 +1659,13 @@ mmc_discover_cards(struct mmc_softc *sc) if (err != MMC_ERR_NONE) { device_printf(sc->dev, "Error selecting card %d\n", err); - break; + goto free_ivar; } err = mmc_app_send_scr(sc, ivar->rca, ivar->raw_scr); if (err != MMC_ERR_NONE) { device_printf(sc->dev, "Error reading SCR %d\n", err); - break; + goto free_ivar; } mmc_app_decode_scr(ivar->raw_scr, &ivar->scr); /* Get card switch capabilities (command class 10). */ @@ -1527,7 +1693,7 @@ mmc_discover_cards(struct mmc_softc *sc) * use from the sd_status is the erase sector size, but * it is still nice to get that right. */ - mmc_select_card(sc, 0); + (void)mmc_select_card(sc, 0); (void)mmc_select_card(sc, ivar->rca); (void)mmc_app_sd_status(sc, ivar->rca, ivar->raw_sd_status); @@ -1537,47 +1703,24 @@ mmc_discover_cards(struct mmc_softc *sc) ivar->erase_sector = 16 << ivar->sd_status.au_size; } - /* Find max supported bus width. */ + /* Find maximum supported bus width. */ if ((host_caps & MMC_CAP_4_BIT_DATA) && (ivar->scr.bus_widths & SD_SCR_BUS_WIDTH_4)) ivar->bus_width = bus_width_4; - /* - * Some cards that report maximum I/O block sizes - * greater than 512 require the block length to be - * set to 512, even though that is supposed to be - * the default. Example: - * - * Transcend 2GB SDSC card, CID: - * mid=0x1b oid=0x534d pnm="00000" prv=1.0 mdt=00.2000 - */ - if (ivar->csd.read_bl_len != MMC_SECTOR_SIZE || - ivar->csd.write_bl_len != MMC_SECTOR_SIZE) - mmc_set_blocklen(sc, MMC_SECTOR_SIZE); - - mmc_format_card_id_string(ivar); - - if (bootverbose || mmc_debug) - mmc_log_card(sc->dev, ivar, newcard); - if (newcard) { - /* Add device. */ - child = device_add_child(sc->dev, NULL, -1); - device_set_ivars(child, ivar); - } - mmc_select_card(sc, 0); - return; + goto child_common; } ivar->rca = rca++; err = mmc_set_relative_addr(sc, ivar->rca); if (err != MMC_ERR_NONE) { device_printf(sc->dev, "Error setting RCA %d\n", err); - break; + goto free_ivar; } /* Get card CSD. */ err = mmc_send_csd(sc, ivar->rca, ivar->raw_csd); if (err != MMC_ERR_NONE) { device_printf(sc->dev, "Error getting CSD %d\n", err); - break; + goto free_ivar; } if (bootverbose || mmc_debug) device_printf(sc->dev, @@ -1596,19 +1739,19 @@ mmc_discover_cards(struct mmc_softc *sc) if (err != MMC_ERR_NONE) { device_printf(sc->dev, "Error reading card status %d\n", err); - break; + goto free_ivar; } if ((status & R1_CARD_IS_LOCKED) != 0) { device_printf(sc->dev, - "Card is password protected, skipping.\n"); - break; + "Card is password protected, skipping\n"); + goto free_ivar; } err = mmc_select_card(sc, ivar->rca); if (err != MMC_ERR_NONE) { device_printf(sc->dev, "Error selecting card %d\n", err); - break; + goto free_ivar; } /* Only MMC >= 4.x devices support EXT_CSD. */ @@ -1618,7 +1761,7 @@ mmc_discover_cards(struct mmc_softc *sc) if (err != MMC_ERR_NONE) { device_printf(sc->dev, "Error reading EXT_CSD %d\n", err); - break; + goto free_ivar; } /* Handle extended capacity from EXT_CSD */ sec_count = ivar->raw_ext_csd[EXT_CSD_SEC_CNT] + @@ -1629,6 +1772,8 @@ mmc_discover_cards(struct mmc_softc *sc) ivar->sec_count = sec_count; ivar->high_cap = 1; } + /* Find maximum supported bus width. */ + ivar->bus_width = mmc_test_bus_width(sc); /* Get device speeds beyond normal mode. */ if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] & EXT_CSD_CARD_TYPE_HS_52) != 0) { @@ -1651,6 +1796,50 @@ mmc_discover_cards(struct mmc_softc *sc) setbit(&ivar->timings, bus_timing_mmc_ddr52); setbit(&ivar->vccq_180, bus_timing_mmc_ddr52); } + if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] & + EXT_CSD_CARD_TYPE_HS200_1_2V) != 0 && + (host_caps & MMC_CAP_SIGNALING_120) != 0) { + setbit(&ivar->timings, bus_timing_mmc_hs200); + setbit(&ivar->vccq_120, bus_timing_mmc_hs200); + } + if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] & + EXT_CSD_CARD_TYPE_HS200_1_8V) != 0 && + (host_caps & MMC_CAP_SIGNALING_180) != 0) { + setbit(&ivar->timings, bus_timing_mmc_hs200); + setbit(&ivar->vccq_180, bus_timing_mmc_hs200); + } + if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] & + EXT_CSD_CARD_TYPE_HS400_1_2V) != 0 && + (host_caps & MMC_CAP_SIGNALING_120) != 0 && + ivar->bus_width == bus_width_8) { + setbit(&ivar->timings, bus_timing_mmc_hs400); + setbit(&ivar->vccq_120, bus_timing_mmc_hs400); + } + if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] & + EXT_CSD_CARD_TYPE_HS400_1_8V) != 0 && + (host_caps & MMC_CAP_SIGNALING_180) != 0 && + ivar->bus_width == bus_width_8) { + setbit(&ivar->timings, bus_timing_mmc_hs400); + setbit(&ivar->vccq_180, bus_timing_mmc_hs400); + } + if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] & + EXT_CSD_CARD_TYPE_HS400_1_2V) != 0 && + (ivar->raw_ext_csd[EXT_CSD_STROBE_SUPPORT] & + EXT_CSD_STROBE_SUPPORT_EN) != 0 && + (host_caps & MMC_CAP_SIGNALING_120) != 0 && + ivar->bus_width == bus_width_8) { + setbit(&ivar->timings, bus_timing_mmc_hs400es); + setbit(&ivar->vccq_120, bus_timing_mmc_hs400es); + } + if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] & + EXT_CSD_CARD_TYPE_HS400_1_8V) != 0 && + (ivar->raw_ext_csd[EXT_CSD_STROBE_SUPPORT] & + EXT_CSD_STROBE_SUPPORT_EN) != 0 && + (host_caps & MMC_CAP_SIGNALING_180) != 0 && + ivar->bus_width == bus_width_8) { + setbit(&ivar->timings, bus_timing_mmc_hs400es); + setbit(&ivar->vccq_180, bus_timing_mmc_hs400es); + } /* * Determine generic switch timeout (provided in * units of 10 ms), defaulting to 500 ms. @@ -1659,8 +1848,6 @@ mmc_discover_cards(struct mmc_softc *sc) if (ivar->csd.spec_vers >= 6) ivar->cmd6_time = 10 * ivar->raw_ext_csd[EXT_CSD_GEN_CMD6_TIME]; - /* Find max supported bus width. */ - ivar->bus_width = mmc_test_bus_width(sc); /* Handle HC erase sector size. */ if (ivar->raw_ext_csd[EXT_CSD_ERASE_GRP_SIZE] != 0) { ivar->erase_sector = 1024 * @@ -1674,11 +1861,15 @@ mmc_discover_cards(struct mmc_softc *sc) device_printf(sc->dev, "Error setting erase group %d\n", err); - break; + goto free_ivar; } } } + mmc_decode_cid_mmc(ivar->raw_cid, &ivar->cid, + ivar->raw_ext_csd[EXT_CSD_REV] >= 5); + +child_common: /* * Some cards that report maximum I/O block sizes greater * than 512 require the block length to be set to 512, even @@ -1691,8 +1882,6 @@ mmc_discover_cards(struct mmc_softc *sc) ivar->csd.write_bl_len != MMC_SECTOR_SIZE) mmc_set_blocklen(sc, MMC_SECTOR_SIZE); - mmc_decode_cid_mmc(ivar->raw_cid, &ivar->cid, - ivar->raw_ext_csd[EXT_CSD_REV] >= 5); mmc_format_card_id_string(ivar); if (bootverbose || mmc_debug) @@ -1700,56 +1889,111 @@ mmc_discover_cards(struct mmc_softc *sc) if (newcard) { /* Add device. */ child = device_add_child(sc->dev, NULL, -1); - device_set_ivars(child, ivar); + if (child != NULL) { + device_set_ivars(child, ivar); + sc->child_list = realloc(sc->child_list, + sizeof(device_t *) * sc->child_count + 1, + M_DEVBUF, M_WAITOK); + sc->child_list[sc->child_count++] = child; + } else + device_printf(sc->dev, "Error adding child\n"); } - mmc_select_card(sc, 0); + +free_ivar: + if (newcard && child == NULL) + free(ivar, M_DEVBUF); + (void)mmc_select_card(sc, 0); + /* + * Not returning here when one MMC device could no be added + * potentially would mean looping forever when that device + * is broken (in which case it also may impact the remainder + * of the bus anyway, though). + */ + if ((newcard && child == NULL) || + mmcbr_get_mode(sc->dev) == mode_sd) + return; } } +static void +mmc_update_child_list(struct mmc_softc *sc) +{ + device_t child; + int i, j; + + if (sc->child_count == 0) { + free(sc->child_list, M_DEVBUF); + return; + } + for (i = j = 0; i < sc->child_count; i++) { + for (;;) { + child = sc->child_list[j++]; + if (child != NULL) + break; + } + if (i != j) + sc->child_list[i] = child; + } + sc->child_list = realloc(sc->child_list, sizeof(device_t *) * + sc->child_count, M_DEVBUF, M_WAITOK); +} + static void mmc_rescan_cards(struct mmc_softc *sc) { struct mmc_ivars *ivar; - device_t *devlist; - int err, i, devcount; + int err, i, j; - if ((err = device_get_children(sc->dev, &devlist, &devcount)) != 0) - return; - for (i = 0; i < devcount; i++) { - ivar = device_get_ivars(devlist[i]); + for (i = j = 0; i < sc->child_count; i++) { + ivar = device_get_ivars(sc->child_list[i]); if (mmc_select_card(sc, ivar->rca) != MMC_ERR_NONE) { if (bootverbose || mmc_debug) device_printf(sc->dev, - "Card at relative address %d lost.\n", + "Card at relative address %d lost\n", ivar->rca); - device_delete_child(sc->dev, devlist[i]); + err = device_delete_child(sc->dev, sc->child_list[i]); + if (err != 0) { + j++; + continue; + } free(ivar, M_DEVBUF); - } + } else + j++; } - free(devlist, M_TEMP); - mmc_select_card(sc, 0); + if (sc->child_count == j) + goto out; + sc->child_count = j; + mmc_update_child_list(sc); +out: + (void)mmc_select_card(sc, 0); } static int -mmc_delete_cards(struct mmc_softc *sc) +mmc_delete_cards(struct mmc_softc *sc, bool final) { struct mmc_ivars *ivar; - device_t *devlist; - int err, i, devcount; + int err, i, j; - if ((err = device_get_children(sc->dev, &devlist, &devcount)) != 0) - return (err); - for (i = 0; i < devcount; i++) { - ivar = device_get_ivars(devlist[i]); + err = 0; + for (i = j = 0; i < sc->child_count; i++) { + ivar = device_get_ivars(sc->child_list[i]); if (bootverbose || mmc_debug) device_printf(sc->dev, - "Card at relative address %d deleted.\n", + "Card at relative address %d deleted\n", ivar->rca); - device_delete_child(sc->dev, devlist[i]); + err = device_delete_child(sc->dev, sc->child_list[i]); + if (err != 0) { + j++; + if (final == false) + continue; + else + break; + } free(ivar, M_DEVBUF); } - free(devlist, M_TEMP); - return (0); + sc->child_count = j; + mmc_update_child_list(sc); + return (err); } static void @@ -1813,7 +2057,7 @@ mmc_go_discovery(struct mmc_softc *sc) mmcbr_get_ocr(dev)); if (mmcbr_get_ocr(dev) == 0) { device_printf(sc->dev, "No compatible cards found on bus\n"); - mmc_delete_cards(sc); + (void)mmc_delete_cards(sc, false); mmc_power_down(sc); return; } @@ -1837,31 +2081,27 @@ mmc_go_discovery(struct mmc_softc *sc) static int mmc_calculate_clock(struct mmc_softc *sc) { - device_t *kids; + device_t dev; struct mmc_ivars *ivar; - int host_caps, i, nkid; + int i; uint32_t dtr, max_dtr; + uint16_t rca; enum mmc_bus_timing max_timing, timing; - bool changed; + bool changed, hs400; - max_dtr = mmcbr_get_f_max(sc->dev); - host_caps = mmcbr_get_caps(sc->dev); - if ((host_caps & MMC_CAP_MMC_DDR52) != 0) - max_timing = bus_timing_mmc_ddr52; - else if ((host_caps & MMC_CAP_HSPEED) != 0) - max_timing = bus_timing_hs; - else - max_timing = bus_timing_normal; - if (device_get_children(sc->dev, &kids, &nkid) != 0) - panic("can't get children"); + dev = sc->dev; + max_dtr = mmcbr_get_f_max(dev); + max_timing = bus_timing_max; do { changed = false; - for (i = 0; i < nkid; i++) { - ivar = device_get_ivars(kids[i]); - if (isclr(&ivar->timings, max_timing)) { - for (timing = max_timing; timing >= + for (i = 0; i < sc->child_count; i++) { + ivar = device_get_ivars(sc->child_list[i]); + if (isclr(&ivar->timings, max_timing) || + !mmc_host_timing(dev, max_timing)) { + for (timing = max_timing - 1; timing >= bus_timing_normal; timing--) { - if (isset(&ivar->timings, timing)) { + if (isset(&ivar->timings, timing) && + mmc_host_timing(dev, timing)) { max_timing = timing; break; } @@ -1875,38 +2115,316 @@ mmc_calculate_clock(struct mmc_softc *sc) } } } while (changed == true); + if (bootverbose || mmc_debug) { - device_printf(sc->dev, + device_printf(dev, "setting transfer rate to %d.%03dMHz (%s timing)\n", max_dtr / 1000000, (max_dtr / 1000) % 1000, mmc_timing_to_string(max_timing)); } - for (i = 0; i < nkid; i++) { - ivar = device_get_ivars(kids[i]); + + /* + * HS400 must be tuned in HS200 mode, so in case of HS400 we begin + * with HS200 following the sequence as described in "6.6.2.2 HS200 + * timing mode selection" of the eMMC specification v5.1, too, and + * switch to max_timing later. HS400ES requires no tuning and, thus, + * can be switch to directly, but requires the same detour via high + * speed mode as does HS400 (see mmc_switch_to_hs400()). + */ + hs400 = max_timing == bus_timing_mmc_hs400; + timing = hs400 == true ? bus_timing_mmc_hs200 : max_timing; + for (i = 0; i < sc->child_count; i++) { + ivar = device_get_ivars(sc->child_list[i]); if ((ivar->timings & ~(1 << bus_timing_normal)) == 0) continue; - if (mmc_select_card(sc, ivar->rca) != MMC_ERR_NONE || - mmc_set_timing(sc, ivar, max_timing) != MMC_ERR_NONE) - device_printf(sc->dev, "Card at relative address %d " - "failed to set timing.\n", ivar->rca); + + rca = ivar->rca; + if (mmc_select_card(sc, rca) != MMC_ERR_NONE) { + device_printf(dev, "Card at relative address %d " + "failed to select\n", rca); + continue; + } + + if (timing == bus_timing_mmc_hs200 || /* includes HS400 */ + timing == bus_timing_mmc_hs400es) { + if (mmc_set_vccq(sc, ivar, timing) != MMC_ERR_NONE) { + device_printf(dev, "Failed to set VCCQ for " + "card at relative address %d\n", rca); + continue; + } + } + + if (timing == bus_timing_mmc_hs200) { /* includes HS400 */ + /* Set bus width (required for initial tuning). */ + if (mmc_set_card_bus_width(sc, ivar, timing) != + MMC_ERR_NONE) { + device_printf(dev, "Card at relative address " + "%d failed to set bus width\n", rca); + continue; + } + mmcbr_set_bus_width(dev, ivar->bus_width); + mmcbr_update_ios(dev); + } else if (timing == bus_timing_mmc_hs400es) { + if (mmc_switch_to_hs400(sc, ivar, max_dtr, timing) != + MMC_ERR_NONE) { + device_printf(dev, "Card at relative address " + "%d failed to set %s timing\n", rca, + mmc_timing_to_string(timing)); + continue; + } + goto power_class; + } + + if (mmc_set_timing(sc, ivar, timing) != MMC_ERR_NONE) { + device_printf(dev, "Card at relative address %d " + "failed to set %s timing\n", rca, + mmc_timing_to_string(timing)); + continue; + } + + if (timing == bus_timing_mmc_ddr52) { + /* + * Set EXT_CSD_BUS_WIDTH_n_DDR in EXT_CSD_BUS_WIDTH + * (must be done after switching to EXT_CSD_HS_TIMING). + */ + if (mmc_set_card_bus_width(sc, ivar, timing) != + MMC_ERR_NONE) { + device_printf(dev, "Card at relative address " + "%d failed to set bus width\n", rca); + continue; + } + mmcbr_set_bus_width(dev, ivar->bus_width); + mmcbr_update_ios(dev); + if (mmc_set_vccq(sc, ivar, timing) != MMC_ERR_NONE) { + device_printf(dev, "Failed to set VCCQ for " + "card at relative address %d\n", rca); + continue; + } + } + + /* Set clock (must be done before initial tuning). */ + mmcbr_set_clock(dev, max_dtr); + mmcbr_update_ios(dev); + + if (mmcbr_tune(dev, hs400) != 0) { + device_printf(dev, "Card at relative address %d " + "failed to execute initial tuning\n", rca); + continue; + } + + if (hs400 == true && mmc_switch_to_hs400(sc, ivar, max_dtr, + max_timing) != MMC_ERR_NONE) { + device_printf(dev, "Card at relative address %d " + "failed to set %s timing\n", rca, + mmc_timing_to_string(max_timing)); + continue; + } + +power_class: + if (mmc_set_power_class(sc, ivar) != MMC_ERR_NONE) { + device_printf(dev, "Card at relative address %d " + "failed to set power class\n", rca); + } } - mmc_select_card(sc, 0); - free(kids, M_TEMP); - mmcbr_set_clock(sc->dev, max_dtr); - mmcbr_update_ios(sc->dev); + (void)mmc_select_card(sc, 0); return (max_dtr); } +/* + * Switch from HS200 to HS400 (either initially or for re-tuning) or directly + * to HS400ES. This follows the sequences described in "6.6.2.3 HS400 timing + * mode selection" of the eMMC specification v5.1. + */ +static int +mmc_switch_to_hs400(struct mmc_softc *sc, struct mmc_ivars *ivar, + uint32_t clock, enum mmc_bus_timing max_timing) +{ + device_t dev; + int err; + uint16_t rca; + + dev = sc->dev; + rca = ivar->rca; + + /* + * Both clock and timing must be set as appropriate for high speed + * before eventually switching to HS400/HS400ES; mmc_set_timing() + * will issue mmcbr_update_ios(). + */ + mmcbr_set_clock(dev, ivar->hs_tran_speed); + err = mmc_set_timing(sc, ivar, bus_timing_hs); + if (err != MMC_ERR_NONE) + return (err); + + /* + * Set EXT_CSD_BUS_WIDTH_8_DDR in EXT_CSD_BUS_WIDTH (and additionally + * EXT_CSD_BUS_WIDTH_ES for HS400ES). + */ + err = mmc_set_card_bus_width(sc, ivar, max_timing); + if (err != MMC_ERR_NONE) + return (err); + mmcbr_set_bus_width(dev, ivar->bus_width); + mmcbr_update_ios(dev); + + /* Finally, switch to HS400/HS400ES mode. */ + err = mmc_set_timing(sc, ivar, max_timing); + if (err != MMC_ERR_NONE) + return (err); + mmcbr_set_clock(dev, clock); + mmcbr_update_ios(dev); + return (MMC_ERR_NONE); +} + +/* + * Switch from HS400 to HS200 (for re-tuning). + */ +static int +mmc_switch_to_hs200(struct mmc_softc *sc, struct mmc_ivars *ivar, + uint32_t clock) +{ + device_t dev; + int err; + uint16_t rca; + + dev = sc->dev; + rca = ivar->rca; + + /* + * Both clock and timing must initially be set as appropriate for + * DDR52 before eventually switching to HS200; mmc_set_timing() + * will issue mmcbr_update_ios(). + */ + mmcbr_set_clock(dev, ivar->hs_tran_speed); + err = mmc_set_timing(sc, ivar, bus_timing_mmc_ddr52); + if (err != MMC_ERR_NONE) + return (err); + + /* + * Next, switch to high speed. Thus, clear EXT_CSD_BUS_WIDTH_n_DDR + * in EXT_CSD_BUS_WIDTH and update bus width and timing in ios. + */ + err = mmc_set_card_bus_width(sc, ivar, bus_timing_hs); + if (err != MMC_ERR_NONE) + return (err); + mmcbr_set_bus_width(dev, ivar->bus_width); + mmcbr_set_timing(sc->dev, bus_timing_hs); + mmcbr_update_ios(dev); + + /* Finally, switch to HS200 mode. */ + err = mmc_set_timing(sc, ivar, bus_timing_mmc_hs200); + if (err != MMC_ERR_NONE) + return (err); + mmcbr_set_clock(dev, clock); + mmcbr_update_ios(dev); + return (MMC_ERR_NONE); +} + +static int +mmc_retune(device_t busdev, device_t dev, bool reset) +{ + struct mmc_softc *sc; + struct mmc_ivars *ivar; + int err; + uint32_t clock; + enum mmc_bus_timing timing; + + if (device_get_parent(dev) != busdev) + return (MMC_ERR_INVALID); + + sc = device_get_softc(busdev); + if (sc->retune_needed != 1 && sc->retune_paused != 0) + return (MMC_ERR_INVALID); + + timing = mmcbr_get_timing(busdev); + if (timing == bus_timing_mmc_hs400) { + /* + * Controllers use the data strobe line to latch data from + * the devices in HS400 mode so periodic re-tuning isn't + * expected to be required, i. e. only if a CRC or tuning + * error is signaled to the bridge. In these latter cases + * we are asked to reset the tuning circuit and need to do + * the switch timing dance. + */ + if (reset == false) + return (0); + ivar = device_get_ivars(dev); + clock = mmcbr_get_clock(busdev); + if (mmc_switch_to_hs200(sc, ivar, clock) != MMC_ERR_NONE) + return (MMC_ERR_BADCRC); + } + err = mmcbr_retune(busdev, reset); + if (err != 0 && timing == bus_timing_mmc_hs400) + return (MMC_ERR_BADCRC); + switch (err) { + case 0: + break; + case EIO: + return (MMC_ERR_FAILED); + default: + return (MMC_ERR_INVALID); + } + if (timing == bus_timing_mmc_hs400) { + if (mmc_switch_to_hs400(sc, ivar, clock, timing) != + MMC_ERR_NONE) + return (MMC_ERR_BADCRC); + } + return (MMC_ERR_NONE); +} + +static void +mmc_retune_pause(device_t busdev, device_t dev, bool retune) +{ + struct mmc_softc *sc; + + sc = device_get_softc(busdev); + KASSERT(device_get_parent(dev) == busdev, + ("%s: %s is not a child of %s", __func__, device_get_nameunit(dev), + device_get_nameunit(busdev))); + KASSERT(sc->owner != NULL, + ("%s: Request from %s without bus being acquired.", __func__, + device_get_nameunit(dev))); + + if (retune == true && sc->retune_paused == 0) + sc->retune_needed = 1; + sc->retune_paused++; +} + +static void +mmc_retune_unpause(device_t busdev, device_t dev) +{ + struct mmc_softc *sc; + + sc = device_get_softc(busdev); + KASSERT(device_get_parent(dev) == busdev, + ("%s: %s is not a child of %s", __func__, device_get_nameunit(dev), + device_get_nameunit(busdev))); + KASSERT(sc->owner != NULL, + ("%s: Request from %s without bus being acquired.", __func__, + device_get_nameunit(dev))); + KASSERT(sc->retune_paused != 0, + ("%s: Re-tune pause count already at 0", __func__)); + + sc->retune_paused--; +} + static void mmc_scan(struct mmc_softc *sc) { device_t dev = sc->dev; + int err; - mmc_acquire_bus(dev, dev); + err = mmc_acquire_bus(dev, dev); + if (err != 0) { + device_printf(dev, "Failed to acquire bus for scanning\n"); + return; + } mmc_go_discovery(sc); - mmc_release_bus(dev, dev); - - bus_generic_attach(dev); + err = mmc_release_bus(dev, dev); + if (err != 0) { + device_printf(dev, "Failed to release bus after scanning\n"); + return; + } + (void)bus_generic_attach(dev); } static int @@ -2005,6 +2523,8 @@ static device_method_t mmc_methods[] = { DEVMETHOD(bus_child_location_str, mmc_child_location_str), /* MMC Bus interface */ + DEVMETHOD(mmcbus_retune_pause, mmc_retune_pause), + DEVMETHOD(mmcbus_retune_unpause, mmc_retune_unpause), DEVMETHOD(mmcbus_wait_for_request, mmc_wait_for_request), DEVMETHOD(mmcbus_acquire_bus, mmc_acquire_bus), DEVMETHOD(mmcbus_release_bus, mmc_release_bus), diff --git a/sys/dev/mmc/mmc_ioctl.h b/sys/dev/mmc/mmc_ioctl.h index 97cff06838c..e633fec9c5e 100644 --- a/sys/dev/mmc/mmc_ioctl.h +++ b/sys/dev/mmc/mmc_ioctl.h @@ -54,7 +54,7 @@ struct mmc_ioc_multi_cmd { #define MMC_IOC_BASE 'M' #define MMC_IOC_CMD _IOWR(MMC_IOC_BASE, 0, struct mmc_ioc_cmd) -#define MMC_IOC_CMD_MULTI _IOWR(MMC_IOC_BASE, 1, struct mmc_ioc_multi_cmd) +#define MMC_IOC_MULTI_CMD _IOWR(MMC_IOC_BASE, 1, struct mmc_ioc_multi_cmd) /* Maximum accepted data transfer size */ #define MMC_IOC_MAX_BYTES (512 * 256) diff --git a/sys/dev/mmc/mmc_private.h b/sys/dev/mmc/mmc_private.h index bbca0c6019f..633d0784d96 100644 --- a/sys/dev/mmc/mmc_private.h +++ b/sys/dev/mmc/mmc_private.h @@ -60,9 +60,14 @@ struct mmc_softc { struct mtx sc_mtx; struct intr_config_hook config_intrhook; device_t owner; - uint32_t last_rca; - int squelched; /* suppress reporting of (expected) errors */ - int log_count; + device_t *child_list; + int child_count; + uint16_t last_rca; + uint16_t retune_paused; + uint8_t retune_needed; + uint8_t retune_ongoing; + uint16_t squelched; /* suppress reporting of (expected) errors */ + int log_count; struct timeval log_time; }; diff --git a/sys/dev/mmc/mmc_subr.c b/sys/dev/mmc/mmc_subr.c index 7f0317a5cf5..af3881c102e 100644 --- a/sys/dev/mmc/mmc_subr.c +++ b/sys/dev/mmc/mmc_subr.c @@ -138,7 +138,6 @@ mmc_wait_for_app_cmd(device_t brdev, device_t reqdev, uint16_t rca, sc->squelched--; if (err != MMC_ERR_NONE && brdev == reqdev) { - sc = device_get_softc(brdev); if (sc->squelched == 0 && ppsratecheck(&sc->log_time, &sc->log_count, LOG_PPS)) { device_printf(sc->dev, "ACMD%d failed, RESULT: %d\n", @@ -154,10 +153,13 @@ mmc_switch(device_t brdev, device_t reqdev, uint16_t rca, uint8_t set, uint8_t index, uint8_t value, u_int timeout, bool status) { struct mmc_command cmd; + struct mmc_softc *sc; int err; KASSERT(timeout != 0, ("%s: no timeout", __func__)); + sc = device_get_softc(brdev); + memset(&cmd, 0, sizeof(cmd)); cmd.opcode = MMC_SWITCH_FUNC; cmd.arg = (MMC_SWITCH_FUNC_WR << 24) | (index << 16) | (value << 8) | @@ -172,10 +174,19 @@ mmc_switch(device_t brdev, device_t reqdev, uint16_t rca, uint8_t set, cmd.flags = MMC_RSP_R1 | MMC_CMD_AC; else cmd.flags = MMC_RSP_R1B | MMC_CMD_AC; + /* + * Pause re-tuning so it won't interfere with the busy state and also + * so that the result of CMD13 will always refer to switching rather + * than to a tuning command that may have snuck in between. + */ + sc->retune_paused++; err = mmc_wait_for_cmd(brdev, reqdev, &cmd, CMD_RETRIES); if (err != MMC_ERR_NONE || status == false) - return (err); - return (mmc_switch_status(brdev, reqdev, rca, timeout)); + goto out; + err = mmc_switch_status(brdev, reqdev, rca, timeout); +out: + sc->retune_paused--; + return (err); } int @@ -192,6 +203,7 @@ mmc_switch_status(device_t brdev, device_t reqdev, uint16_t rca, u_int timeout) * type MMC_CAP_WAIT_WHILE_BUSY will issue mmc_send_status() only * once and then exit the loop. */ + end.tv_sec = end.tv_usec = 0; for (;;) { err = mmc_send_status(brdev, reqdev, rca, &status); if (err != MMC_ERR_NONE) @@ -208,7 +220,7 @@ mmc_switch_status(device_t brdev, device_t reqdev, uint16_t rca, u_int timeout) break; } } - if (err == MMC_ERR_NONE && R1_CURRENT_STATE(status) == R1_SWITCH_ERROR) + if (err == MMC_ERR_NONE && (status & R1_SWITCH_ERROR) != 0) return (MMC_ERR_FAILED); return (err); } diff --git a/sys/dev/mmc/mmcbr_if.m b/sys/dev/mmc/mmcbr_if.m index 34ffdb031ed..497ff3acca2 100644 --- a/sys/dev/mmc/mmcbr_if.m +++ b/sys/dev/mmc/mmcbr_if.m @@ -54,7 +54,6 @@ # #include -#include #include # @@ -74,6 +73,22 @@ CODE { return (0); } + + static int + null_retune(device_t brdev __unused, device_t reqdev __unused, + bool reset __unused) + { + + return (0); + } + + static int + null_tune(device_t brdev __unused, device_t reqdev __unused, + bool hs400 __unused) + { + + return (0); + } }; # @@ -94,11 +109,30 @@ METHOD int switch_vccq { device_t reqdev; } DEFAULT null_switch_vccq; +# +# Called by the mmcbus with the bridge claimed to execute initial tuning. +# +METHOD int tune { + device_t brdev; + device_t reqdev; + bool hs400; +} DEFAULT null_tune; + +# +# Called by the mmcbus with the bridge claimed to execute re-tuning. +# +METHOD int retune { + device_t brdev; + device_t reqdev; + bool reset; +} DEFAULT null_retune; + # # Called by the mmcbus or its children to schedule a mmc request. These # requests are queued. Time passes. The bridge then gets notification # of the status of the request, who then notifies the requesting device # by calling the completion function supplied as part of the request. +# Requires the bridge to be claimed. # METHOD int request { device_t brdev; @@ -121,7 +155,7 @@ METHOD int get_ro { METHOD int acquire_host { device_t brdev; device_t reqdev; -} +}; # # Release the current bridge. @@ -129,4 +163,4 @@ METHOD int acquire_host { METHOD int release_host { device_t brdev; device_t reqdev; -} +}; diff --git a/sys/dev/mmc/mmcbrvar.h b/sys/dev/mmc/mmcbrvar.h index 47024da7696..c8cf0020e2b 100644 --- a/sys/dev/mmc/mmcbrvar.h +++ b/sys/dev/mmc/mmcbrvar.h @@ -56,6 +56,7 @@ #define DEV_MMC_MMCBRVAR_H #include + #include "mmcbr_if.h" enum mmcbr_device_ivars { @@ -69,6 +70,7 @@ enum mmcbr_device_ivars { MMCBR_IVAR_MODE, MMCBR_IVAR_OCR, MMCBR_IVAR_POWER_MODE, + MMCBR_IVAR_RETUNE_REQ, MMCBR_IVAR_VDD, MMCBR_IVAR_VCCQ, MMCBR_IVAR_CAPS, @@ -93,6 +95,7 @@ MMCBR_ACCESSOR(host_ocr, HOST_OCR, int) MMCBR_ACCESSOR(mode, MODE, int) MMCBR_ACCESSOR(ocr, OCR, int) MMCBR_ACCESSOR(power_mode, POWER_MODE, int) +MMCBR_ACCESSOR(retune_req, RETUNE_REQ, int) MMCBR_ACCESSOR(vdd, VDD, int) MMCBR_ACCESSOR(vccq, VCCQ, int) MMCBR_ACCESSOR(caps, CAPS, int) @@ -107,6 +110,20 @@ mmcbr_update_ios(device_t dev) return (MMCBR_UPDATE_IOS(device_get_parent(dev), dev)); } +static int __inline +mmcbr_tune(device_t dev, bool hs400) +{ + + return (MMCBR_TUNE(device_get_parent(dev), dev, hs400)); +} + +static int __inline +mmcbr_retune(device_t dev, bool reset) +{ + + return (MMCBR_RETUNE(device_get_parent(dev), dev, reset)); +} + static int __inline mmcbr_switch_vccq(device_t dev) { diff --git a/sys/dev/mmc/mmcbus_if.m b/sys/dev/mmc/mmcbus_if.m index 14dc8e92f8c..d33510d507b 100644 --- a/sys/dev/mmc/mmcbus_if.m +++ b/sys/dev/mmc/mmcbus_if.m @@ -53,18 +53,36 @@ # $FreeBSD$ # +#include #include -#include # -# This is the set of callbacks that mmc bridges call into the bus, or -# that mmc/sd card drivers call to make requests. +# This is the set of callbacks that the MMC subroutines and the mmc/sd card +# driver call into the bus to make requests. # INTERFACE mmcbus; # -# Queue and wait for a request. +# Pause re-tuning, optionally with triggering re-tuning up-front. Requires +# the bus to be claimed. +# +METHOD void retune_pause { + device_t brdev; + device_t reqdev; + bool retune; +}; + +# +# Unpause re-tuning. Requires the bus to be claimed. +# +METHOD void retune_unpause { + device_t brdev; + device_t reqdev; +}; + +# +# Queue and wait for a request. Requires the bus to be claimed. # METHOD int wait_for_request { device_t brdev; @@ -73,18 +91,18 @@ METHOD int wait_for_request { }; # -# Claim the current bridge, blocking the current thread until the host -# is no longer busy. +# Claim the current bus, blocking the current thread until the host is no +# longer busy. # METHOD int acquire_bus { device_t brdev; device_t reqdev; -} +}; # -# Release the current bridge. +# Release the current bus. # METHOD int release_bus { device_t brdev; device_t reqdev; -} +}; diff --git a/sys/dev/mmc/mmcreg.h b/sys/dev/mmc/mmcreg.h index 671b8c23529..82c47731446 100644 --- a/sys/dev/mmc/mmcreg.h +++ b/sys/dev/mmc/mmcreg.h @@ -204,6 +204,7 @@ struct mmc_request { void *done_data; /* requestor set data */ uint32_t flags; #define MMC_REQ_DONE 1 +#define MMC_TUNE_DONE 2 }; /* Command definitions */ @@ -431,8 +432,8 @@ struct mmc_request { #define EXT_CSD_HS_TIMING_BC 0 #define EXT_CSD_HS_TIMING_HS 1 -#define EXT_CSD_HS_TIMING_DDR200 2 -#define EXT_CSD_HS_TIMING_DDR400 3 +#define EXT_CSD_HS_TIMING_HS200 2 +#define EXT_CSD_HS_TIMING_HS400 3 #define EXT_CSD_HS_TIMING_DRV_STR_SHIFT 4 #define EXT_CSD_POWER_CLASS_8BIT_MASK 0xf0 @@ -448,7 +449,6 @@ struct mmc_request { #define EXT_CSD_CARD_TYPE_HS200_1_2V 0x0020 #define EXT_CSD_CARD_TYPE_HS400_1_8V 0x0040 #define EXT_CSD_CARD_TYPE_HS400_1_2V 0x0080 -#define EXT_CSD_CARD_TYPE_HS400ES 0x0100 #define EXT_CSD_BUS_WIDTH_1 0 #define EXT_CSD_BUS_WIDTH_4 1 @@ -457,6 +457,8 @@ struct mmc_request { #define EXT_CSD_BUS_WIDTH_8_DDR 6 #define EXT_CSD_BUS_WIDTH_ES 0x80 +#define EXT_CSD_STROBE_SUPPORT_EN 0x01 + #define MMC_TYPE_HS_26_MAX 26000000 #define MMC_TYPE_HS_52_MAX 52000000 #define MMC_TYPE_DDR52_MAX 52000000 @@ -657,6 +659,10 @@ struct mmc_sd_status #define MMC_PART_GP_MAX 4 #define MMC_PART_MAX 8 +#define MMC_TUNING_MAX 64 /* Maximum tuning iterations */ +#define MMC_TUNING_LEN 64 /* Size of tuning data */ +#define MMC_TUNING_LEN_HS200 128 /* Size of tuning data in HS200 mode */ + /* * Older versions of the MMC standard had a variable sector size. However, * I've been able to find no old MMC or SD cards that have a non 512 diff --git a/sys/dev/mmc/mmcsd.c b/sys/dev/mmc/mmcsd.c index 4b2e29f1181..a04ff587175 100644 --- a/sys/dev/mmc/mmcsd.c +++ b/sys/dev/mmc/mmcsd.c @@ -99,7 +99,8 @@ __FBSDID("$FreeBSD$"); struct mmcsd_softc; struct mmcsd_part { - struct mtx part_mtx; + struct mtx disk_mtx; + struct mtx ioctl_mtx; struct mmcsd_softc *sc; struct disk *disk; struct proc *p; @@ -109,6 +110,7 @@ struct mmcsd_part { u_int type; int running; int suspend; + int ioctl; bool ro; char name[MMCSD_PART_NAMELEN]; }; @@ -118,6 +120,9 @@ struct mmcsd_softc { device_t mmcbr; struct mmcsd_part *part[MMC_PART_MAX]; enum mmc_card_mode mode; + u_int max_data; /* Maximum data size [blocks] */ + u_int erase_sector; /* Device native erase sector size [blocks] */ + uint8_t high_cap; /* High Capacity device (block addressed) */ uint8_t part_curr; /* Partition currently switched to */ uint8_t ext_csd[MMC_EXTCSD_SIZE]; uint16_t rca; @@ -178,15 +183,25 @@ static int mmcsd_slicer(device_t dev, const char *provider, static int mmcsd_switch_part(device_t bus, device_t dev, uint16_t rca, u_int part); -#define MMCSD_PART_LOCK(_part) mtx_lock(&(_part)->part_mtx) -#define MMCSD_PART_UNLOCK(_part) mtx_unlock(&(_part)->part_mtx) -#define MMCSD_PART_LOCK_INIT(_part) \ - mtx_init(&(_part)->part_mtx, (_part)->name, "mmcsd part", MTX_DEF) -#define MMCSD_PART_LOCK_DESTROY(_part) mtx_destroy(&(_part)->part_mtx); -#define MMCSD_PART_ASSERT_LOCKED(_part) \ - mtx_assert(&(_part)->part_mtx, MA_OWNED); -#define MMCSD_PART_ASSERT_UNLOCKED(_part) \ - mtx_assert(&(_part)->part_mtx, MA_NOTOWNED); +#define MMCSD_DISK_LOCK(_part) mtx_lock(&(_part)->disk_mtx) +#define MMCSD_DISK_UNLOCK(_part) mtx_unlock(&(_part)->disk_mtx) +#define MMCSD_DISK_LOCK_INIT(_part) \ + mtx_init(&(_part)->disk_mtx, (_part)->name, "mmcsd disk", MTX_DEF) +#define MMCSD_DISK_LOCK_DESTROY(_part) mtx_destroy(&(_part)->disk_mtx); +#define MMCSD_DISK_ASSERT_LOCKED(_part) \ + mtx_assert(&(_part)->disk_mtx, MA_OWNED); +#define MMCSD_DISK_ASSERT_UNLOCKED(_part) \ + mtx_assert(&(_part)->disk_mtx, MA_NOTOWNED); + +#define MMCSD_IOCTL_LOCK(_part) mtx_lock(&(_part)->ioctl_mtx) +#define MMCSD_IOCTL_UNLOCK(_part) mtx_unlock(&(_part)->ioctl_mtx) +#define MMCSD_IOCTL_LOCK_INIT(_part) \ + mtx_init(&(_part)->ioctl_mtx, (_part)->name, "mmcsd IOCTL", MTX_DEF) +#define MMCSD_IOCTL_LOCK_DESTROY(_part) mtx_destroy(&(_part)->ioctl_mtx); +#define MMCSD_IOCTL_ASSERT_LOCKED(_part) \ + mtx_assert(&(_part)->ioctl_mtx, MA_OWNED); +#define MMCSD_IOCLT_ASSERT_UNLOCKED(_part) \ + mtx_assert(&(_part)->ioctl_mtx, MA_NOTOWNED); static int mmcsd_probe(device_t dev) @@ -214,6 +229,18 @@ mmcsd_attach(device_t dev) sc->dev = dev; sc->mmcbr = mmcbr = device_get_parent(dev); sc->mode = mmcbr_get_mode(mmcbr); + /* + * Note that in principle with an SDHCI-like re-tuning implementation, + * the maximum data size can change at runtime due to a device removal/ + * insertion that results in switches to/from a transfer mode involving + * re-tuning, iff there are multiple devices on a given bus. Until now + * mmc(4) lacks support for rescanning already attached buses, however, + * and sdhci(4) has no support for embedded/shared buses in the first + * place either. + */ + sc->max_data = mmc_get_max_data(dev); + sc->erase_sector = mmc_get_erase_sector(dev); + sc->high_cap = mmc_get_high_cap(dev); sc->rca = mmc_get_rca(dev); /* Only MMC >= 4.x devices support EXT_CSD. */ @@ -267,7 +294,7 @@ mmcsd_attach(device_t dev) (ext_csd[EXT_CSD_ENH_START_ADDR + 1] << 8) + (ext_csd[EXT_CSD_ENH_START_ADDR + 2] << 16) + (ext_csd[EXT_CSD_ENH_START_ADDR + 3] << 24)) * - (mmc_get_high_cap(dev) ? MMC_SECTOR_SIZE : 1); + (sc->high_cap != 0 ? MMC_SECTOR_SIZE : 1); } else if (bootverbose) device_printf(dev, "enhanced user data area spans entire device\n"); @@ -280,7 +307,7 @@ mmcsd_attach(device_t dev) ro = mmc_get_read_only(dev); mmcsd_add_part(sc, EXT_CSD_PART_CONFIG_ACC_DEFAULT, "mmcsd", device_get_unit(dev), mmc_get_media_size(dev) * sector_size, - mmc_get_erase_sector(dev) * sector_size, ro); + sc->erase_sector * sector_size, ro); if (mmc_get_spec_vers(dev) < 3) return (0); @@ -417,7 +444,16 @@ mmcsd_add_part(struct mmcsd_softc *sc, u_int type, const char *name, u_int cnt, part->ro = ro; snprintf(part->name, sizeof(part->name), name, device_get_unit(dev)); - /* For the RPMB partition, allow IOCTL access only. */ + MMCSD_IOCTL_LOCK_INIT(part); + + /* + * For the RPMB partition, allow IOCTL access only. + * NB: If ever attaching RPMB partitions to disk(9), the re-tuning + * implementation and especially its pausing need to be revisited, + * because then re-tuning requests may be issued by the IOCTL half + * of this driver while re-tuning is already paused by the disk(9) + * one and vice versa. + */ if (type == EXT_CSD_PART_CONFIG_ACC_RPMB) { make_dev_args_init(&args); args.mda_flags = MAKEDEV_CHECKNAME | MAKEDEV_WAITOK; @@ -432,7 +468,7 @@ mmcsd_add_part(struct mmcsd_softc *sc, u_int type, const char *name, u_int cnt, return; } } else { - MMCSD_PART_LOCK_INIT(part); + MMCSD_DISK_LOCK_INIT(part); d = part->disk = disk_alloc(); d->d_open = mmcsd_open; @@ -444,7 +480,7 @@ mmcsd_add_part(struct mmcsd_softc *sc, u_int type, const char *name, u_int cnt, d->d_name = part->name; d->d_drv1 = part; d->d_sectorsize = mmc_get_sector_size(dev); - d->d_maxsize = mmc_get_max_data(dev) * d->d_sectorsize; + d->d_maxsize = sc->max_data * d->d_sectorsize; d->d_mediasize = media_size; d->d_stripesize = erase_size; d->d_unit = cnt; @@ -471,7 +507,7 @@ mmcsd_add_part(struct mmcsd_softc *sc, u_int type, const char *name, u_int cnt, part->name, cnt, bytes, unit, mmc_get_card_id_string(dev), ro ? " (read-only)" : "", device_get_nameunit(mmcbr), speed / 1000000, (speed / 100000) % 10, - mmcsd_bus_bit_width(dev), mmc_get_max_data(dev)); + mmcsd_bus_bit_width(dev), sc->max_data); } else if (type == EXT_CSD_PART_CONFIG_ACC_RPMB) { printf("%s: %ju%sB partion %d%s at %s\n", part->name, bytes, unit, type, ro ? " (read-only)" : "", @@ -559,19 +595,27 @@ mmcsd_detach(device_t dev) for (i = 0; i < MMC_PART_MAX; i++) { part = sc->part[i]; - if (part != NULL && part->disk != NULL) { - MMCSD_PART_LOCK(part); - part->suspend = 0; - if (part->running > 0) { - /* kill thread */ - part->running = 0; - wakeup(part); - /* wait for thread to finish. */ - while (part->running != -1) - msleep(part, &part->part_mtx, 0, - "detach", 0); + if (part != NULL) { + if (part->disk != NULL) { + MMCSD_DISK_LOCK(part); + part->suspend = 0; + if (part->running > 0) { + /* kill thread */ + part->running = 0; + wakeup(part); + /* wait for thread to finish. */ + while (part->running != -1) + msleep(part, &part->disk_mtx, 0, + "mmcsd disk detach", 0); + } + MMCSD_DISK_UNLOCK(part); } - MMCSD_PART_UNLOCK(part); + MMCSD_IOCTL_LOCK(part); + while (part->ioctl > 0) + msleep(part, &part->ioctl_mtx, 0, + "mmcsd IOCTL detach", 0); + part->ioctl = -1; + MMCSD_IOCTL_UNLOCK(part); } } @@ -587,8 +631,9 @@ mmcsd_detach(device_t dev) /* kill disk */ disk_destroy(part->disk); - MMCSD_PART_LOCK_DESTROY(part); + MMCSD_DISK_LOCK_DESTROY(part); } + MMCSD_IOCTL_LOCK_DESTROY(part); free(part, M_DEVBUF); } } @@ -604,19 +649,27 @@ mmcsd_suspend(device_t dev) for (i = 0; i < MMC_PART_MAX; i++) { part = sc->part[i]; - if (part != NULL && part->disk != NULL) { - MMCSD_PART_LOCK(part); - part->suspend = 1; - if (part->running > 0) { - /* kill thread */ - part->running = 0; - wakeup(part); - /* wait for thread to finish. */ - while (part->running != -1) - msleep(part, &part->part_mtx, 0, - "detach", 0); + if (part != NULL) { + if (part->disk != NULL) { + MMCSD_DISK_LOCK(part); + part->suspend = 1; + if (part->running > 0) { + /* kill thread */ + part->running = 0; + wakeup(part); + /* wait for thread to finish. */ + while (part->running != -1) + msleep(part, &part->disk_mtx, 0, + "mmcsd disk suspension", 0); + } + MMCSD_DISK_UNLOCK(part); } - MMCSD_PART_UNLOCK(part); + MMCSD_IOCTL_LOCK(part); + while (part->ioctl > 0) + msleep(part, &part->ioctl_mtx, 0, + "mmcsd IOCTL suspension", 0); + part->ioctl = -1; + MMCSD_IOCTL_UNLOCK(part); } } return (0); @@ -631,16 +684,22 @@ mmcsd_resume(device_t dev) for (i = 0; i < MMC_PART_MAX; i++) { part = sc->part[i]; - if (part != NULL && part->disk != NULL) { - MMCSD_PART_LOCK(part); - part->suspend = 0; - if (part->running <= 0) { - part->running = 1; - kproc_create(&mmcsd_task, part, &part->p, 0, 0, - "%s%d: mmc/sd card", part->name, part->cnt); - MMCSD_PART_UNLOCK(part); - } else - MMCSD_PART_UNLOCK(part); + if (part != NULL) { + if (part->disk != NULL) { + MMCSD_DISK_LOCK(part); + part->suspend = 0; + if (part->running <= 0) { + part->running = 1; + MMCSD_DISK_UNLOCK(part); + kproc_create(&mmcsd_task, part, + &part->p, 0, 0, "%s%d: mmc/sd card", + part->name, part->cnt); + } else + MMCSD_DISK_UNLOCK(part); + } + MMCSD_IOCTL_LOCK(part); + part->ioctl = 0; + MMCSD_IOCTL_UNLOCK(part); } } return (0); @@ -668,13 +727,13 @@ mmcsd_strategy(struct bio *bp) part = bp->bio_disk->d_drv1; sc = part->sc; - MMCSD_PART_LOCK(part); + MMCSD_DISK_LOCK(part); if (part->running > 0 || part->suspend > 0) { bioq_disksort(&part->bio_queue, bp); - MMCSD_PART_UNLOCK(part); + MMCSD_DISK_UNLOCK(part); wakeup(part); } else { - MMCSD_PART_UNLOCK(part); + MMCSD_DISK_UNLOCK(part); biofinish(bp, NULL, ENXIO); } } @@ -710,9 +769,9 @@ mmcsd_ioctl(struct mmcsd_part *part, u_long cmd, void *data, int fflag) switch (cmd) { case MMC_IOC_CMD: mic = data; - err = mmcsd_ioctl_cmd(part, data, fflag); + err = mmcsd_ioctl_cmd(part, mic, fflag); break; - case MMC_IOC_CMD_MULTI: + case MMC_IOC_MULTI_CMD: mimc = data; if (mimc->num_of_cmds == 0) break; @@ -722,12 +781,12 @@ mmcsd_ioctl(struct mmcsd_part *part, u_long cmd, void *data, int fflag) size = sizeof(*mic) * cnt; mic = malloc(size, M_TEMP, M_WAITOK); err = copyin((const void *)mimc->cmds, mic, size); - if (err != 0) - break; - for (i = 0; i < cnt; i++) { - err = mmcsd_ioctl_cmd(part, &mic[i], fflag); - if (err != 0) - break; + if (err == 0) { + for (i = 0; i < cnt; i++) { + err = mmcsd_ioctl_cmd(part, &mic[i], fflag); + if (err != 0) + break; + } } free(mic, M_TEMP); break; @@ -756,11 +815,31 @@ mmcsd_ioctl_cmd(struct mmcsd_part *part, struct mmc_ioc_cmd *mic, int fflag) if (part->ro == TRUE && mic->write_flag != 0) return (EROFS); + /* + * We don't need to explicitly lock against the disk(9) half of this + * driver as MMCBUS_ACQUIRE_BUS() will serialize us. However, it's + * necessary to protect against races with detachment and suspension, + * especially since it's required to switch away from RPMB partitions + * again after an access (see mmcsd_switch_part()). + */ + MMCSD_IOCTL_LOCK(part); + while (part->ioctl != 0) { + if (part->ioctl < 0) { + MMCSD_IOCTL_UNLOCK(part); + return (ENXIO); + } + msleep(part, &part->ioctl_mtx, 0, "mmcsd IOCTL", 0); + } + part->ioctl = 1; + MMCSD_IOCTL_UNLOCK(part); + err = 0; dp = NULL; len = mic->blksz * mic->blocks; - if (len > MMC_IOC_MAX_BYTES) - return (EOVERFLOW); + if (len > MMC_IOC_MAX_BYTES) { + err = EOVERFLOW; + goto out; + } if (len != 0) { dp = malloc(len, M_TEMP, M_WAITOK); err = copyin((void *)(uintptr_t)mic->data_ptr, dp, len); @@ -815,7 +894,7 @@ mmcsd_ioctl_cmd(struct mmcsd_part *part, struct mmc_ioc_cmd *mic, int fflag) err = mmcsd_set_blockcount(sc, mic->blocks, mic->write_flag & (1 << 31)); if (err != MMC_ERR_NONE) - goto release; + goto switch_back; } if (mic->is_acmd != 0) (void)mmc_wait_for_app_cmd(mmcbr, dev, rca, &cmd, 0); @@ -837,6 +916,7 @@ mmcsd_ioctl_cmd(struct mmcsd_part *part, struct mmc_ioc_cmd *mic, int fflag) DELAY(1000); } while (retries-- > 0); +switch_back: /* ... and always switch back to the default partition. */ err = mmcsd_switch_part(mmcbr, dev, rca, EXT_CSD_PART_CONFIG_ACC_DEFAULT); @@ -887,6 +967,10 @@ release: err = EIO; out: + MMCSD_IOCTL_LOCK(part); + part->ioctl = 0; + MMCSD_IOCTL_UNLOCK(part); + wakeup(part); if (dp != NULL) free(dp, M_TEMP); return (err); @@ -938,10 +1022,23 @@ mmcsd_switch_part(device_t bus, device_t dev, uint16_t rca, u_int part) sc = device_get_softc(dev); - if (sc->part_curr == part) + if (sc->mode == mode_sd) return (MMC_ERR_NONE); - if (sc->mode == mode_sd) + /* + * According to section "6.2.2 Command restrictions" of the eMMC + * specification v5.1, CMD19/CMD21 aren't allowed to be used with + * RPMB partitions. So we pause re-tuning along with triggering + * it up-front to decrease the likelihood of re-tuning becoming + * necessary while accessing an RPMB partition. Consequently, an + * RPMB partition should immediately be switched away from again + * after an access in order to allow for re-tuning to take place + * anew. + */ + if (part == EXT_CSD_PART_CONFIG_ACC_RPMB) + MMCBUS_RETUNE_PAUSE(sc->mmcbr, sc->dev, true); + + if (sc->part_curr == part) return (MMC_ERR_NONE); value = (sc->ext_csd[EXT_CSD_PART_CONFIG] & @@ -949,10 +1046,15 @@ mmcsd_switch_part(device_t bus, device_t dev, uint16_t rca, u_int part) /* Jump! */ err = mmc_switch(bus, dev, rca, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_PART_CONFIG, value, sc->part_time, true); - if (err != MMC_ERR_NONE) + if (err != MMC_ERR_NONE) { + if (part == EXT_CSD_PART_CONFIG_ACC_RPMB) + MMCBUS_RETUNE_UNPAUSE(sc->mmcbr, sc->dev); return (err); + } sc->ext_csd[EXT_CSD_PART_CONFIG] = value; + if (sc->part_curr == EXT_CSD_PART_CONFIG_ACC_RPMB) + MMCBUS_RETUNE_UNPAUSE(sc->mmcbr, sc->dev); sc->part_curr = part; return (MMC_ERR_NONE); } @@ -963,7 +1065,7 @@ mmcsd_errmsg(int e) if (e < 0 || e > MMC_ERR_MAX) return "Bad error code"; - return errmsg[e]; + return (errmsg[e]); } static daddr_t @@ -976,7 +1078,7 @@ mmcsd_rw(struct mmcsd_part *part, struct bio *bp) struct mmc_data data; struct mmcsd_softc *sc; device_t dev, mmcbr; - int numblocks, sz; + u_int numblocks, sz; char *vaddr; sc = part->sc; @@ -988,7 +1090,7 @@ mmcsd_rw(struct mmcsd_part *part, struct bio *bp) end = bp->bio_pblkno + (bp->bio_bcount / sz); while (block < end) { vaddr = bp->bio_data + (block - bp->bio_pblkno) * sz; - numblocks = min(end - block, mmc_get_max_data(dev)); + numblocks = min(end - block, sc->max_data); memset(&req, 0, sizeof(req)); memset(&cmd, 0, sizeof(cmd)); memset(&stop, 0, sizeof(stop)); @@ -1008,7 +1110,7 @@ mmcsd_rw(struct mmcsd_part *part, struct bio *bp) cmd.opcode = MMC_WRITE_BLOCK; } cmd.arg = block; - if (!mmc_get_high_cap(dev)) + if (sc->high_cap == 0) cmd.arg <<= 9; cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC; data.data = vaddr; @@ -1048,7 +1150,7 @@ mmcsd_delete(struct mmcsd_part *part, struct bio *bp) struct mmc_request req; struct mmcsd_softc *sc; device_t dev, mmcbr; - int erase_sector, sz; + u_int erase_sector, sz; sc = part->sc; dev = sc->dev; @@ -1063,7 +1165,7 @@ mmcsd_delete(struct mmcsd_part *part, struct bio *bp) if (end >= part->eblock && end < part->eend) end = part->eend; /* Safe round to the erase sector boundaries. */ - erase_sector = mmc_get_erase_sector(dev); + erase_sector = sc->erase_sector; start = block + erase_sector - 1; /* Round up. */ start -= start % erase_sector; stop = end; /* Round down. */ @@ -1075,6 +1177,12 @@ mmcsd_delete(struct mmcsd_part *part, struct bio *bp) return (end); } + /* + * Pause re-tuning so it won't interfere with the order of erase + * commands. Note that these latter don't use the data lines, so + * re-tuning shouldn't actually become necessary during erase. + */ + MMCBUS_RETUNE_PAUSE(mmcbr, dev, false); /* Set erase start position. */ memset(&req, 0, sizeof(req)); memset(&cmd, 0, sizeof(cmd)); @@ -1085,13 +1193,15 @@ mmcsd_delete(struct mmcsd_part *part, struct bio *bp) else cmd.opcode = MMC_ERASE_GROUP_START; cmd.arg = start; - if (!mmc_get_high_cap(dev)) + if (sc->high_cap == 0) cmd.arg <<= 9; cmd.flags = MMC_RSP_R1 | MMC_CMD_AC; MMCBUS_WAIT_FOR_REQUEST(mmcbr, dev, &req); if (req.cmd->error != MMC_ERR_NONE) { - printf("erase err1: %d\n", req.cmd->error); - return (block); + device_printf(dev, "Setting erase start position failed %d\n", + req.cmd->error); + block = bp->bio_pblkno; + goto unpause; } /* Set erase stop position. */ memset(&req, 0, sizeof(req)); @@ -1102,14 +1212,16 @@ mmcsd_delete(struct mmcsd_part *part, struct bio *bp) else cmd.opcode = MMC_ERASE_GROUP_END; cmd.arg = stop; - if (!mmc_get_high_cap(dev)) + if (sc->high_cap == 0) cmd.arg <<= 9; cmd.arg--; cmd.flags = MMC_RSP_R1 | MMC_CMD_AC; MMCBUS_WAIT_FOR_REQUEST(mmcbr, dev, &req); if (req.cmd->error != MMC_ERR_NONE) { - printf("erase err2: %d\n", req.cmd->error); - return (block); + device_printf(dev, "Setting erase stop position failed %d\n", + req.cmd->error); + block = bp->bio_pblkno; + goto unpause; } /* Erase range. */ memset(&req, 0, sizeof(req)); @@ -1120,8 +1232,11 @@ mmcsd_delete(struct mmcsd_part *part, struct bio *bp) cmd.flags = MMC_RSP_R1B | MMC_CMD_AC; MMCBUS_WAIT_FOR_REQUEST(mmcbr, dev, &req); if (req.cmd->error != MMC_ERR_NONE) { - printf("erase err3 %d\n", req.cmd->error); - return (block); + device_printf(dev, "erase err3: %d\n", req.cmd->error); + device_printf(dev, "Issuing erase command failed %d\n", + req.cmd->error); + block = bp->bio_pblkno; + goto unpause; } /* Store one of remaining parts for the next call. */ if (bp->bio_pblkno >= part->eblock || block == start) { @@ -1131,7 +1246,10 @@ mmcsd_delete(struct mmcsd_part *part, struct bio *bp) part->eblock = block; /* Predict next backward. */ part->eend = start; } - return (end); + block = end; +unpause: + MMCBUS_RETUNE_UNPAUSE(mmcbr, dev); + return (block); } static int @@ -1192,16 +1310,16 @@ mmcsd_task(void *arg) mmcbr = sc->mmcbr; while (1) { - MMCSD_PART_LOCK(part); + MMCSD_DISK_LOCK(part); do { if (part->running == 0) goto out; bp = bioq_takefirst(&part->bio_queue); if (bp == NULL) - msleep(part, &part->part_mtx, PRIBIO, - "jobqueue", 0); + msleep(part, &part->disk_mtx, PRIBIO, + "mmcsd disk jobqueue", 0); } while (bp == NULL); - MMCSD_PART_UNLOCK(part); + MMCSD_DISK_UNLOCK(part); if (bp->bio_cmd != BIO_READ && part->ro) { bp->bio_error = EROFS; bp->bio_resid = bp->bio_bcount; @@ -1242,7 +1360,7 @@ release: out: /* tell parent we're done */ part->running = -1; - MMCSD_PART_UNLOCK(part); + MMCSD_DISK_UNLOCK(part); wakeup(part); kproc_exit(0); diff --git a/sys/dev/sdhci/sdhci.c b/sys/dev/sdhci/sdhci.c index e6bb25969c3..3df8bde7fe4 100644 --- a/sys/dev/sdhci/sdhci.c +++ b/sys/dev/sdhci/sdhci.c @@ -1,5 +1,6 @@ /*- * Copyright (c) 2008 Alexander Motin + * Copyright (c) 2017 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -32,7 +33,9 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include +#include #include #include #include @@ -48,6 +51,8 @@ __FBSDID("$FreeBSD$"); #include #include +#include + #include #include #include @@ -55,7 +60,6 @@ __FBSDID("$FreeBSD$"); #include #include "mmcbr_if.h" -#include "sdhci.h" #include "sdhci_if.h" #include "opt_mmccam.h" @@ -84,13 +88,15 @@ SYSCTL_INT(_hw_sdhci, OID_AUTO, quirk_set, CTLFLAG_RWTUN, &sdhci_quirk_set, 0, #define WR_MULTI_4(slot, off, ptr, count) \ SDHCI_WRITE_MULTI_4((slot)->bus, (slot), (off), (ptr), (count)) +static void sdhci_card_poll(void *arg); +static void sdhci_card_task(void *arg, int pending); +static int sdhci_exec_tuning(struct sdhci_slot *slot, bool reset); +static void sdhci_req_wakeup(struct mmc_request *req); +static void sdhci_retune(void *arg); static void sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock); static void sdhci_start(struct sdhci_slot *slot); static void sdhci_start_data(struct sdhci_slot *slot, struct mmc_data *data); -static void sdhci_card_poll(void *); -static void sdhci_card_task(void *, int); - #ifdef MMCCAM /* CAM-related */ int sdhci_cam_get_possible_host_clock(struct sdhci_slot *slot, int proposed_clock); @@ -105,6 +111,7 @@ static int sdhci_cam_settran_settings(struct sdhci_slot *slot, union ccb *ccb); static void sdhci_dumpregs(struct sdhci_slot *slot); static int slot_printf(struct sdhci_slot *slot, const char * fmt, ...) __printflike(2, 3); +static uint32_t sdhci_tuning_intmask(struct sdhci_slot *slot); #define SDHCI_LOCK(_slot) mtx_lock(&(_slot)->mtx) #define SDHCI_UNLOCK(_slot) mtx_unlock(&(_slot)->mtx) @@ -185,13 +192,13 @@ sdhci_dumpregs(struct sdhci_slot *slot) RD1(slot, SDHCI_TIMEOUT_CONTROL), RD4(slot, SDHCI_INT_STATUS)); slot_printf(slot, "Int enab: 0x%08x | Sig enab: 0x%08x\n", RD4(slot, SDHCI_INT_ENABLE), RD4(slot, SDHCI_SIGNAL_ENABLE)); - slot_printf(slot, "AC12 err: 0x%08x | Host ctl2: 0x%08x\n", + slot_printf(slot, "AC12 err: 0x%08x | Host ctl2:0x%08x\n", RD2(slot, SDHCI_ACMD12_ERR), RD2(slot, SDHCI_HOST_CONTROL2)); slot_printf(slot, "Caps: 0x%08x | Caps2: 0x%08x\n", RD4(slot, SDHCI_CAPABILITIES), RD4(slot, SDHCI_CAPABILITIES2)); slot_printf(slot, "Max curr: 0x%08x | ADMA err: 0x%08x\n", RD4(slot, SDHCI_MAX_CURRENT), RD1(slot, SDHCI_ADMA_ERR)); - slot_printf(slot, "ADMA addr: 0x%08x | Slot int: 0x%08x\n", + slot_printf(slot, "ADMA addr:0x%08x | Slot int: 0x%08x\n", RD4(slot, SDHCI_ADMA_ADDRESS_LO), RD2(slot, SDHCI_SLOT_INT_STATUS)); slot_printf(slot, @@ -258,6 +265,21 @@ sdhci_reset(struct sdhci_slot *slot, uint8_t mask) } } +static uint32_t +sdhci_tuning_intmask(struct sdhci_slot *slot) +{ + uint32_t intmask; + + intmask = 0; + if (slot->opt & SDHCI_TUNING_SUPPORTED) { + intmask |= SDHCI_INT_TUNEERR; + if (slot->retune_mode == SDHCI_RETUNE_MODE_2 || + slot->retune_mode == SDHCI_RETUNE_MODE_3) + intmask |= SDHCI_INT_RETUNE; + } + return (intmask); +} + static void sdhci_init(struct sdhci_slot *slot) { @@ -277,7 +299,7 @@ sdhci_init(struct sdhci_slot *slot) slot->intmask |= SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT; } - WR4(slot, SDHCI_INT_ENABLE, slot->intmask); + WR4(slot, SDHCI_INT_ENABLE, slot->intmask | sdhci_tuning_intmask(slot)); WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask); } @@ -583,10 +605,12 @@ sdhci_card_task(void *arg, int pending __unused) SDHCI_UNLOCK(slot); xpt_rescan(ccb); #else - slot->dev = device_add_child(slot->bus, "mmc", -1); - device_set_ivars(slot->dev, slot); + d = slot->dev = device_add_child(slot->bus, "mmc", -1); SDHCI_UNLOCK(slot); - device_probe_and_attach(slot->dev); + if (d) { + device_set_ivars(d, slot); + (void)device_probe_and_attach(d); + } #endif } else SDHCI_UNLOCK(slot); @@ -627,7 +651,11 @@ sdhci_card_task(void *arg, int pending __unused) SDHCI_UNLOCK(slot); xpt_rescan(ccb); #else + slot->intmask &= ~sdhci_tuning_intmask(slot); + WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask); + slot->opt &= ~SDHCI_TUNING_ENABLED; SDHCI_UNLOCK(slot); + callout_drain(&slot->retune_callout); device_delete_child(slot->bus, d); #endif } else @@ -687,6 +715,8 @@ sdhci_card_poll(void *arg) int sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num) { + kobjop_desc_t kobj_desc; + kobj_method_t *kobj_method; uint32_t caps, caps2, freq, host_caps; int err; @@ -728,8 +758,6 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num) return (EFAULT); } - /* Initialize slot. */ - sdhci_init(slot); slot->version = (RD2(slot, SDHCI_HOST_VERSION) >> SDHCI_SPEC_VER_SHIFT) & SDHCI_SPEC_VER_MASK; if (slot->quirks & SDHCI_QUIRK_MISSING_CAPS) { @@ -797,6 +825,7 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num) device_printf(dev, "Hardware doesn't report any " "support voltages.\n"); } + host_caps = MMC_CAP_4_BIT_DATA; if (caps & SDHCI_CAN_DO_8BITBUS) host_caps |= MMC_CAP_8_BIT_DATA; @@ -806,6 +835,8 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num) host_caps |= MMC_CAP_BOOT_NOACC; if (slot->quirks & SDHCI_QUIRK_WAIT_WHILE_BUSY) host_caps |= MMC_CAP_WAIT_WHILE_BUSY; + + /* Determine supported UHS-I and eMMC modes. */ if (caps2 & (SDHCI_CAN_SDR50 | SDHCI_CAN_SDR104 | SDHCI_CAN_DDR50)) host_caps |= MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25; if (caps2 & SDHCI_CAN_SDR104) { @@ -822,12 +853,91 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num) if (slot->quirks & SDHCI_QUIRK_CAPS_BIT63_FOR_MMC_HS400 && caps2 & SDHCI_CAN_MMC_HS400) host_caps |= MMC_CAP_MMC_HS400; + + /* + * Disable UHS-I and eMMC modes if the set_uhs_timing method is the + * default NULL implementation. + */ + kobj_desc = &sdhci_set_uhs_timing_desc; + kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL, + kobj_desc); + if (kobj_method == &kobj_desc->deflt) + host_caps &= ~(MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 | + MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_DDR50 | MMC_CAP_UHS_SDR104 | + MMC_CAP_MMC_DDR52 | MMC_CAP_MMC_HS200 | MMC_CAP_MMC_HS400); + +#define SDHCI_CAP_MODES_TUNING(caps2) \ + (((caps2) & SDHCI_TUNE_SDR50 ? MMC_CAP_UHS_SDR50 : 0) | \ + MMC_CAP_UHS_DDR50 | MMC_CAP_UHS_SDR104 | MMC_CAP_MMC_HS200 | \ + MMC_CAP_MMC_HS400) + + /* + * Disable UHS-I and eMMC modes that require (re-)tuning if either + * the tune or re-tune method is the default NULL implementation. + */ + kobj_desc = &mmcbr_tune_desc; + kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL, + kobj_desc); + if (kobj_method == &kobj_desc->deflt) + goto no_tuning; + kobj_desc = &mmcbr_retune_desc; + kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL, + kobj_desc); + if (kobj_method == &kobj_desc->deflt) { +no_tuning: + host_caps &= ~(SDHCI_CAP_MODES_TUNING(caps2)); + } + + /* Allocate tuning structures and determine tuning parameters. */ + if (host_caps & SDHCI_CAP_MODES_TUNING(caps2)) { + slot->opt |= SDHCI_TUNING_SUPPORTED; + slot->tune_req = malloc(sizeof(*slot->tune_req), M_DEVBUF, + M_WAITOK); + slot->tune_cmd = malloc(sizeof(*slot->tune_cmd), M_DEVBUF, + M_WAITOK); + slot->tune_data = malloc(sizeof(*slot->tune_data), M_DEVBUF, + M_WAITOK); + if (caps2 & SDHCI_TUNE_SDR50) + slot->opt |= SDHCI_SDR50_NEEDS_TUNING; + slot->retune_mode = (caps2 & SDHCI_RETUNE_MODES_MASK) >> + SDHCI_RETUNE_MODES_SHIFT; + if (slot->retune_mode == SDHCI_RETUNE_MODE_1) { + slot->retune_count = (caps2 & SDHCI_RETUNE_CNT_MASK) >> + SDHCI_RETUNE_CNT_SHIFT; + if (slot->retune_count > 0xb) { + device_printf(dev, "Unknown re-tuning count " + "%x, using 1 sec\n", slot->retune_count); + slot->retune_count = 1; + } else if (slot->retune_count != 0) + slot->retune_count = + 1 << (slot->retune_count - 1); + } + } + +#undef SDHCI_CAP_MODES_TUNING + + /* Determine supported VCCQ signaling levels. */ host_caps |= MMC_CAP_SIGNALING_330; if (host_caps & (MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 | - MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104 | MMC_CAP_UHS_DDR50 | + MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_DDR50 | MMC_CAP_UHS_SDR104 | MMC_CAP_MMC_DDR52_180 | MMC_CAP_MMC_HS200_180 | MMC_CAP_MMC_HS400_180)) - host_caps |= MMC_CAP_SIGNALING_180; + host_caps |= MMC_CAP_SIGNALING_120 | MMC_CAP_SIGNALING_180; + + /* + * Disable 1.2 V and 1.8 V signaling if the switch_vccq method is the + * default NULL implementation. Disable 1.2 V support if it's the + * generic SDHCI implementation. + */ + kobj_desc = &mmcbr_switch_vccq_desc; + kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL, + kobj_desc); + if (kobj_method == &kobj_desc->deflt) + host_caps &= ~(MMC_CAP_SIGNALING_120 | MMC_CAP_SIGNALING_180); + else if (kobj_method->func == (kobjop_t)sdhci_generic_switch_vccq) + host_caps &= ~MMC_CAP_SIGNALING_120; + + /* Determine supported driver types (type B is always mandatory). */ if (caps2 & SDHCI_CAN_DRIVE_TYPE_A) host_caps |= MMC_CAP_DRIVER_TYPE_A; if (caps2 & SDHCI_CAN_DRIVE_TYPE_C) @@ -866,9 +976,9 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num) (caps & SDHCI_CAN_VDD_180) ? " 1.8V" : "", (host_caps & MMC_CAP_SIGNALING_180) ? " 1.8V" : "", (host_caps & MMC_CAP_SIGNALING_120) ? " 1.2V" : "", - (caps2 & SDHCI_CAN_DRIVE_TYPE_A) ? "A" : "", - (caps2 & SDHCI_CAN_DRIVE_TYPE_C) ? "C" : "", - (caps2 & SDHCI_CAN_DRIVE_TYPE_D) ? "D" : "", + (host_caps & MMC_CAP_DRIVER_TYPE_A) ? "A" : "", + (host_caps & MMC_CAP_DRIVER_TYPE_C) ? "C" : "", + (host_caps & MMC_CAP_DRIVER_TYPE_D) ? "D" : "", (slot->opt & SDHCI_HAVE_DMA) ? "DMA" : "PIO"); if (host_caps & (MMC_CAP_MMC_DDR52 | MMC_CAP_MMC_HS200 | MMC_CAP_MMC_HS400 | MMC_CAP_MMC_ENH_STROBE)) @@ -888,6 +998,9 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num) (host_caps & MMC_CAP_UHS_SDR50) ? " SDR50" : "", (host_caps & MMC_CAP_UHS_SDR104) ? " SDR104" : "", (host_caps & MMC_CAP_UHS_DDR50) ? " DDR50" : ""); + if (slot->opt & SDHCI_TUNING_SUPPORTED) + slot_printf(slot, "Re-tuning count %d secs, mode %d\n", + slot->retune_count, slot->retune_mode + 1); sdhci_dumpregs(slot); } @@ -901,6 +1014,7 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num) sdhci_card_task, slot); callout_init(&slot->card_poll_callout, 1); callout_init_mtx(&slot->timeout_callout, &slot->mtx, 0); + callout_init_mtx(&slot->retune_callout, &slot->mtx, 0); if ((slot->quirks & SDHCI_QUIRK_POLL_CARD_PRESENT) && !(slot->opt & SDHCI_NON_REMOVABLE)) { @@ -908,6 +1022,8 @@ sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num) SDHCI_CARD_PRESENT_TICKS, sdhci_card_poll, slot); } + sdhci_init(slot); + return (0); } @@ -925,6 +1041,7 @@ sdhci_cleanup_slot(struct sdhci_slot *slot) callout_drain(&slot->timeout_callout); callout_drain(&slot->card_poll_callout); + callout_drain(&slot->retune_callout); taskqueue_drain(taskqueue_swi_giant, &slot->card_task); taskqueue_drain_timeout(taskqueue_swi_giant, &slot->card_delayed_task); @@ -941,6 +1058,11 @@ sdhci_cleanup_slot(struct sdhci_slot *slot) bus_dmamap_unload(slot->dmatag, slot->dmamap); bus_dmamem_free(slot->dmatag, slot->dmamem, slot->dmamap); bus_dma_tag_destroy(slot->dmatag); + if (slot->opt & SDHCI_TUNING_SUPPORTED) { + free(slot->tune_req, M_DEVBUF); + free(slot->tune_cmd, M_DEVBUF); + free(slot->tune_data, M_DEVBUF); + } SDHCI_LOCK_DESTROY(slot); @@ -951,7 +1073,16 @@ int sdhci_generic_suspend(struct sdhci_slot *slot) { + /* + * We expect the MMC layer to issue initial tuning after resume. + * Otherwise, we'd need to indicate re-tuning including circuit reset + * being required at least for re-tuning modes 1 and 2 ourselves. + */ + callout_drain(&slot->retune_callout); + SDHCI_LOCK(slot); + slot->opt &= ~SDHCI_TUNING_ENABLED; sdhci_reset(slot, SDHCI_RESET_ALL); + SDHCI_UNLOCK(slot); return (0); } @@ -960,7 +1091,9 @@ int sdhci_generic_resume(struct sdhci_slot *slot) { + SDHCI_LOCK(slot); sdhci_init(slot); + SDHCI_UNLOCK(slot); return (0); } @@ -994,15 +1127,18 @@ sdhci_generic_set_uhs_timing(device_t brdev __unused, struct sdhci_slot *slot) if (slot->version < SDHCI_SPEC_300) return; + SDHCI_ASSERT_LOCKED(slot); ios = &slot->host.ios; sdhci_set_clock(slot, 0); hostctrl2 = RD2(slot, SDHCI_HOST_CONTROL2); hostctrl2 &= ~SDHCI_CTRL2_UHS_MASK; - if (ios->timing == bus_timing_mmc_hs400 || - ios->timing == bus_timing_mmc_hs400es) - hostctrl2 |= SDHCI_CTRL2_MMC_HS400; - else if (ios->clock > SD_SDR50_MAX) - hostctrl2 |= SDHCI_CTRL2_UHS_SDR104; + if (ios->clock > SD_SDR50_MAX) { + if (ios->timing == bus_timing_mmc_hs400 || + ios->timing == bus_timing_mmc_hs400es) + hostctrl2 |= SDHCI_CTRL2_MMC_HS400; + else + hostctrl2 |= SDHCI_CTRL2_UHS_SDR104; + } else if (ios->clock > SD_SDR25_MAX) hostctrl2 |= SDHCI_CTRL2_UHS_SDR50; else if (ios->clock > SD_SDR12_MAX) { @@ -1114,13 +1250,196 @@ done: return (err); } +int +sdhci_generic_tune(device_t brdev __unused, device_t reqdev, bool hs400) +{ + struct sdhci_slot *slot = device_get_ivars(reqdev); + struct mmc_ios *ios = &slot->host.ios; + struct mmc_command *tune_cmd; + struct mmc_data *tune_data; + uint32_t opcode; + int err; + + if (!(slot->opt & SDHCI_TUNING_SUPPORTED)) + return (0); + + slot->retune_ticks = slot->retune_count * hz; + opcode = MMC_SEND_TUNING_BLOCK; + SDHCI_LOCK(slot); + switch (ios->timing) { + case bus_timing_mmc_hs400: + slot_printf(slot, "HS400 must be tuned in HS200 mode\n"); + SDHCI_UNLOCK(slot); + return (EINVAL); + case bus_timing_mmc_hs200: + /* + * In HS400 mode, controllers use the data strobe line to + * latch data from the devices so periodic re-tuning isn't + * expected to be required. + */ + if (hs400) + slot->retune_ticks = 0; + opcode = MMC_SEND_TUNING_BLOCK_HS200; + break; + case bus_timing_uhs_ddr50: + case bus_timing_uhs_sdr104: + break; + case bus_timing_uhs_sdr50: + if (slot->opt & SDHCI_SDR50_NEEDS_TUNING) + break; + /* FALLTHROUGH */ + default: + SDHCI_UNLOCK(slot); + return (0); + } + + tune_cmd = slot->tune_cmd; + memset(tune_cmd, 0, sizeof(*tune_cmd)); + tune_cmd->opcode = opcode; + tune_cmd->flags = MMC_RSP_R1 | MMC_CMD_ADTC; + tune_data = tune_cmd->data = slot->tune_data; + memset(tune_data, 0, sizeof(*tune_data)); + tune_data->len = (opcode == MMC_SEND_TUNING_BLOCK_HS200 && + ios->bus_width == bus_width_8) ? MMC_TUNING_LEN_HS200 : + MMC_TUNING_LEN; + tune_data->flags = MMC_DATA_READ; + tune_data->mrq = tune_cmd->mrq = slot->tune_req; + + slot->opt &= ~SDHCI_TUNING_ENABLED; + err = sdhci_exec_tuning(slot, true); + if (err == 0) { + slot->opt |= SDHCI_TUNING_ENABLED; + slot->intmask |= sdhci_tuning_intmask(slot); + WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask); + if (slot->retune_ticks) { + callout_reset(&slot->retune_callout, slot->retune_ticks, + sdhci_retune, slot); + } + } + SDHCI_UNLOCK(slot); + return (err); +} + +int +sdhci_generic_retune(device_t brdev __unused, device_t reqdev, bool reset) +{ + struct sdhci_slot *slot = device_get_ivars(reqdev); + int err; + + if (!(slot->opt & SDHCI_TUNING_ENABLED)) + return (0); + + /* HS400 must be tuned in HS200 mode. */ + if (slot->host.ios.timing == bus_timing_mmc_hs400) + return (EINVAL); + + SDHCI_LOCK(slot); + err = sdhci_exec_tuning(slot, reset); + /* + * There are two ways sdhci_exec_tuning() can fail: + * EBUSY should not actually happen when requests are only issued + * with the host properly acquired, and + * EIO re-tuning failed (but it did work initially). + * + * In both cases, we should retry at later point if periodic re-tuning + * is enabled. Note that due to slot->retune_req not being cleared in + * these failure cases, the MMC layer should trigger another attempt at + * re-tuning with the next request anyway, though. + */ + if (slot->retune_ticks) { + callout_reset(&slot->retune_callout, slot->retune_ticks, + sdhci_retune, slot); + } + SDHCI_UNLOCK(slot); + return (err); +} + +static int +sdhci_exec_tuning(struct sdhci_slot *slot, bool reset) +{ + struct mmc_request *tune_req; + struct mmc_command *tune_cmd; + int i; + uint32_t intmask; + uint16_t hostctrl2; + u_char opt; + + SDHCI_ASSERT_LOCKED(slot); + if (slot->req != NULL) + return (EBUSY); + + /* Tuning doesn't work with DMA enabled. */ + opt = slot->opt; + slot->opt = opt & ~SDHCI_HAVE_DMA; + + /* + * Ensure that as documented, SDHCI_INT_DATA_AVAIL is the only + * kind of interrupt we receive in response to a tuning request. + */ + intmask = slot->intmask; + slot->intmask = SDHCI_INT_DATA_AVAIL; + WR4(slot, SDHCI_SIGNAL_ENABLE, SDHCI_INT_DATA_AVAIL); + + hostctrl2 = RD2(slot, SDHCI_HOST_CONTROL2); + if (reset) + hostctrl2 &= ~SDHCI_CTRL2_SAMPLING_CLOCK; + else + hostctrl2 |= SDHCI_CTRL2_SAMPLING_CLOCK; + WR2(slot, SDHCI_HOST_CONTROL2, hostctrl2 | SDHCI_CTRL2_EXEC_TUNING); + + tune_req = slot->tune_req; + tune_cmd = slot->tune_cmd; + for (i = 0; i < MMC_TUNING_MAX; i++) { + memset(tune_req, 0, sizeof(*tune_req)); + tune_req->cmd = tune_cmd; + tune_req->done = sdhci_req_wakeup; + tune_req->done_data = slot; + slot->req = tune_req; + slot->flags = 0; + sdhci_start(slot); + while (!(tune_req->flags & MMC_REQ_DONE)) + msleep(tune_req, &slot->mtx, 0, "sdhciet", 0); + if (!(tune_req->flags & MMC_TUNE_DONE)) + break; + hostctrl2 = RD2(slot, SDHCI_HOST_CONTROL2); + if (!(hostctrl2 & SDHCI_CTRL2_EXEC_TUNING)) + break; + if (tune_cmd->opcode == MMC_SEND_TUNING_BLOCK) + DELAY(1000); + } + + slot->opt = opt; + slot->intmask = intmask; + WR4(slot, SDHCI_SIGNAL_ENABLE, intmask); + + if ((hostctrl2 & (SDHCI_CTRL2_EXEC_TUNING | + SDHCI_CTRL2_SAMPLING_CLOCK)) == SDHCI_CTRL2_SAMPLING_CLOCK) { + slot->retune_req = 0; + return (0); + } + + slot_printf(slot, "Tuning failed, using fixed sampling clock\n"); + WR2(slot, SDHCI_HOST_CONTROL2, hostctrl2 & ~(SDHCI_CTRL2_EXEC_TUNING | + SDHCI_CTRL2_SAMPLING_CLOCK)); + sdhci_reset(slot, SDHCI_RESET_CMD | SDHCI_RESET_DATA); + return (EIO); +} + +static void +sdhci_retune(void *arg) +{ + struct sdhci_slot *slot = arg; + + slot->retune_req |= SDHCI_RETUNE_REQ_NEEDED; +} + #ifdef MMCCAM static void sdhci_req_done(struct sdhci_slot *slot) { union ccb *ccb; - if (sdhci_debug > 1) + if (__predict_false(sdhci_debug > 1)) slot_printf(slot, "%s\n", __func__); if (slot->ccb != NULL && slot->curcmd != NULL) { callout_stop(&slot->timeout_callout); @@ -1153,19 +1472,29 @@ sdhci_req_done(struct sdhci_slot *slot) } #endif +static void +sdhci_req_wakeup(struct mmc_request *req) +{ + struct sdhci_slot *slot; + + slot = req->done_data; + req->flags |= MMC_REQ_DONE; + wakeup(req); +} + static void sdhci_timeout(void *arg) { struct sdhci_slot *slot = arg; if (slot->curcmd != NULL) { - slot_printf(slot, " Controller timeout\n"); + slot_printf(slot, "Controller timeout\n"); sdhci_dumpregs(slot); sdhci_reset(slot, SDHCI_RESET_CMD | SDHCI_RESET_DATA); slot->curcmd->error = MMC_ERR_TIMEOUT; sdhci_req_done(slot); } else { - slot_printf(slot, " Spurious timeout - no active command\n"); + slot_printf(slot, "Spurious timeout - no active command\n"); } } @@ -1236,15 +1565,19 @@ sdhci_start_command(struct sdhci_slot *slot, struct mmc_command *cmd) /* Wait for free DAT if we have data or busy signal. */ if (cmd->data != NULL || (cmd->flags & MMC_RSP_BUSY)) mask |= SDHCI_DAT_INHIBIT; - /* We shouldn't wait for DAT for stop commands. */ + /* + * We shouldn't wait for DAT for stop commands or CMD19/CMD21. Note + * that these latter are also special in that SDHCI_CMD_DATA should + * be set below but no actual data is ever read from the controller. + */ #ifdef MMCCAM - struct ccb_mmcio *mmcio = &slot->ccb->mmcio; - if (cmd == &mmcio->stop) - mask &= ~SDHCI_DAT_INHIBIT; + if (cmd == &slot->ccb->mmcio.stop || #else - if (cmd == slot->req->stop) - mask &= ~SDHCI_DAT_INHIBIT; + if (cmd == slot->req->stop || #endif + __predict_false(cmd->opcode == MMC_SEND_TUNING_BLOCK || + cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200)) + mask &= ~SDHCI_DAT_INHIBIT; /* * Wait for bus no more then 250 ms. Typically there will be no wait * here at all, but when writing a crash dump we may be bypassing the @@ -1301,7 +1634,7 @@ sdhci_start_command(struct sdhci_slot *slot, struct mmc_command *cmd) WR4(slot, SDHCI_ARGUMENT, cmd->arg); /* Set data transfer mode. */ sdhci_set_transfer_mode(slot, cmd->data); - if (sdhci_debug > 1) + if (__predict_false(sdhci_debug > 1)) slot_printf(slot, "Starting command!\n"); /* Start command. */ WR2(slot, SDHCI_COMMAND_FLAGS, (cmd->opcode << 8) | (flags & 0xff)); @@ -1317,7 +1650,7 @@ sdhci_finish_command(struct sdhci_slot *slot) uint32_t val; uint8_t extra; - if (sdhci_debug > 1) + if (__predict_false(sdhci_debug > 1)) slot_printf(slot, "%s: called, err %d flags %d\n", __func__, slot->curcmd->error, slot->curcmd->flags); slot->cmd_done = 1; @@ -1326,9 +1659,14 @@ sdhci_finish_command(struct sdhci_slot *slot) * Main restore point for the case when command interrupt * happened first. */ - WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask |= SDHCI_INT_RESPONSE); + if (__predict_true(slot->curcmd->opcode != MMC_SEND_TUNING_BLOCK && + slot->curcmd->opcode != MMC_SEND_TUNING_BLOCK_HS200)) + WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask |= + SDHCI_INT_RESPONSE); /* In case of error - reset host and return. */ if (slot->curcmd->error) { + if (slot->curcmd->error == MMC_ERR_BADCRC) + slot->retune_req |= SDHCI_RETUNE_REQ_RESET; sdhci_reset(slot, SDHCI_RESET_CMD); sdhci_reset(slot, SDHCI_RESET_DATA); sdhci_start(slot); @@ -1353,7 +1691,7 @@ sdhci_finish_command(struct sdhci_slot *slot) } else slot->curcmd->resp[0] = RD4(slot, SDHCI_RESPONSE); } - if (sdhci_debug > 1) + if (__predict_false(sdhci_debug > 1)) printf("Resp: %02x %02x %02x %02x\n", slot->curcmd->resp[0], slot->curcmd->resp[1], slot->curcmd->resp[2], slot->curcmd->resp[3]); @@ -1439,7 +1777,7 @@ sdhci_start_data(struct sdhci_slot *slot, struct mmc_data *data) /* Set block count. */ WR2(slot, SDHCI_BLOCK_COUNT, (data->len + 511) / 512); - if (sdhci_debug > 1) + if (__predict_false(sdhci_debug > 1)) slot_printf(slot, "Block size: %02x, count %lu\n", (unsigned int)SDHCI_MAKE_BLKSZ(DMA_BOUNDARY, (data->len < 512) ? data->len : 512), (unsigned long)(data->len + 511) / 512); @@ -1474,6 +1812,8 @@ sdhci_finish_data(struct sdhci_slot *slot) slot->data_done = 1; /* If there was error - reset the host. */ if (slot->curcmd->error) { + if (slot->curcmd->error == MMC_ERR_BADCRC) + slot->retune_req |= SDHCI_RETUNE_REQ_RESET; sdhci_reset(slot, SDHCI_RESET_CMD); sdhci_reset(slot, SDHCI_RESET_DATA); sdhci_start(slot); @@ -1514,7 +1854,7 @@ sdhci_start(struct sdhci_slot *slot) return; } */ - if (sdhci_debug > 1) + if (__predict_false(sdhci_debug > 1)) slot_printf(slot, "result: %d\n", mmcio->cmd.error); if (mmcio->cmd.error == 0 && (slot->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST)) { @@ -1545,7 +1885,7 @@ sdhci_start(struct sdhci_slot *slot) sdhci_start_command(slot, req->stop); return; } - if (sdhci_debug > 1) + if (__predict_false(sdhci_debug > 1)) slot_printf(slot, "result: %d\n", req->cmd->error); if (!req->cmd->error && ((slot->curcmd == req->stop && @@ -1570,7 +1910,7 @@ sdhci_generic_request(device_t brdev __unused, device_t reqdev, SDHCI_UNLOCK(slot); return (EBUSY); } - if (sdhci_debug > 1) { + if (__predict_false(sdhci_debug > 1)) { slot_printf(slot, "CMD%u arg %#x flags %#x dlen %u dflags %#x\n", req->cmd->opcode, req->cmd->arg, req->cmd->flags, @@ -1689,6 +2029,15 @@ sdhci_data_irq(struct sdhci_slot *slot, uint32_t intmask) goto done; } + /* Handle tuning completion interrupt. */ + if (__predict_false((intmask & SDHCI_INT_DATA_AVAIL) && + (slot->curcmd->opcode == MMC_SEND_TUNING_BLOCK || + slot->curcmd->opcode == MMC_SEND_TUNING_BLOCK_HS200))) { + slot->req->flags |= MMC_TUNE_DONE; + sdhci_finish_command(slot); + sdhci_finish_data(slot); + return; + } /* Handle PIO interrupt. */ if (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL)) { if ((slot->opt & SDHCI_PLATFORM_TRANSFER) && @@ -1781,9 +2130,21 @@ sdhci_generic_intr(struct sdhci_slot *slot) SDHCI_UNLOCK(slot); return; } - if (sdhci_debug > 2) + if (__predict_false(sdhci_debug > 2)) slot_printf(slot, "Interrupt %#x\n", intmask); + /* Handle tuning error interrupt. */ + if (__predict_false(intmask & SDHCI_INT_TUNEERR)) { + slot_printf(slot, "Tuning error indicated\n"); + slot->retune_req |= SDHCI_RETUNE_REQ_RESET; + if (slot->curcmd) { + slot->curcmd->error = MMC_ERR_BADCRC; + sdhci_finish_command(slot); + } + } + /* Handle re-tuning interrupt. */ + if (__predict_false(intmask & SDHCI_INT_RETUNE)) + slot->retune_req |= SDHCI_RETUNE_REQ_NEEDED; /* Handle card presence interrupts. */ if (intmask & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) { present = (intmask & SDHCI_INT_CARD_INSERT) != 0; @@ -1796,7 +2157,6 @@ sdhci_generic_intr(struct sdhci_slot *slot) WR4(slot, SDHCI_INT_STATUS, intmask & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)); sdhci_handle_card_present_locked(slot, present); - intmask &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE); } /* Handle command interrupts. */ if (intmask & SDHCI_INT_CMD_MASK) { @@ -1815,17 +2175,14 @@ sdhci_generic_intr(struct sdhci_slot *slot) WR4(slot, SDHCI_INT_STATUS, SDHCI_INT_ACMD12ERR); sdhci_acmd_irq(slot); } - intmask &= ~(SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK); - intmask &= ~SDHCI_INT_ACMD12ERR; - intmask &= ~SDHCI_INT_ERROR; /* Handle bus power interrupt. */ if (intmask & SDHCI_INT_BUS_POWER) { WR4(slot, SDHCI_INT_STATUS, SDHCI_INT_BUS_POWER); - slot_printf(slot, - "Card is consuming too much power!\n"); - intmask &= ~SDHCI_INT_BUS_POWER; + slot_printf(slot, "Card is consuming too much power!\n"); } - + intmask &= ~(SDHCI_INT_ERROR | SDHCI_INT_TUNEERR | SDHCI_INT_RETUNE | + SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE | SDHCI_INT_CMD_MASK | + SDHCI_INT_DATA_MASK | SDHCI_INT_ACMD12ERR | SDHCI_INT_BUS_POWER); /* The rest is unknown. */ if (intmask) { WR4(slot, SDHCI_INT_STATUS, intmask); @@ -1879,6 +2236,19 @@ sdhci_generic_read_ivar(device_t bus, device_t child, int which, case MMCBR_IVAR_VDD: *result = slot->host.ios.vdd; break; + case MMCBR_IVAR_RETUNE_REQ: + if (slot->opt & SDHCI_TUNING_ENABLED) { + if (slot->retune_req & SDHCI_RETUNE_REQ_RESET) { + *result = retune_req_reset; + break; + } + if (slot->retune_req & SDHCI_RETUNE_REQ_NEEDED) { + *result = retune_req_normal; + break; + } + } + *result = retune_req_none; + break; case MMCBR_IVAR_VCCQ: *result = slot->host.ios.vccq; break; @@ -1889,6 +2259,16 @@ sdhci_generic_read_ivar(device_t bus, device_t child, int which, *result = slot->host.ios.timing; break; case MMCBR_IVAR_MAX_DATA: + /* + * Re-tuning modes 1 and 2 restrict the maximum data length + * per read/write command to 4 MiB. + */ + if (slot->opt & SDHCI_TUNING_ENABLED && + (slot->retune_mode == SDHCI_RETUNE_MODE_1 || + slot->retune_mode == SDHCI_RETUNE_MODE_2)) { + *result = 4 * 1024 * 1024 / MMC_SECTOR_SIZE; + break; + } *result = 65535; break; case MMCBR_IVAR_MAX_BUSY_TIMEOUT: @@ -1971,19 +2351,13 @@ sdhci_generic_write_ivar(device_t bus, device_t child, int which, case MMCBR_IVAR_F_MIN: case MMCBR_IVAR_F_MAX: case MMCBR_IVAR_MAX_DATA: + case MMCBR_IVAR_RETUNE_REQ: return (EINVAL); } return (0); } #ifdef MMCCAM -/* CAM-related functions */ -#include -#include -#include -#include -#include - void sdhci_cam_start_slot(struct sdhci_slot *slot) { @@ -2123,7 +2497,7 @@ sdhci_cam_action(struct cam_sim *sim, union ccb *ccb) * At some point in the future an interrupt comes. * Then the request will be marked as completed. */ - if (sdhci_debug > 1) + if (__predict_false(sdhci_debug > 1)) slot_printf(slot, "Got XPT_MMC_IO\n"); ccb->ccb_h.status = CAM_REQ_INPROG; @@ -2272,7 +2646,7 @@ sdhci_cam_request(struct sdhci_slot *slot, union ccb *ccb) return (EBUSY); } */ - if (sdhci_debug > 1) { + if (__predict_false(sdhci_debug > 1)) { slot_printf(slot, "CMD%u arg %#x flags %#x dlen %u dflags %#x\n", mmcio->cmd.opcode, mmcio->cmd.arg, mmcio->cmd.flags, mmcio->cmd.data != NULL ? (unsigned int) mmcio->cmd.data->len : 0, diff --git a/sys/dev/sdhci/sdhci.h b/sys/dev/sdhci/sdhci.h index 52d17600917..cd5f75e6a51 100644 --- a/sys/dev/sdhci/sdhci.h +++ b/sys/dev/sdhci/sdhci.h @@ -328,6 +328,7 @@ extern u_int sdhci_quirk_clear; extern u_int sdhci_quirk_set; struct sdhci_slot { + struct mtx mtx; /* Slot mutex */ u_int quirks; /* Chip specific quirks */ u_int caps; /* Override SDHCI_CAPABILITIES */ u_int caps2; /* Override SDHCI_CAPABILITIES2 */ @@ -338,6 +339,9 @@ struct sdhci_slot { #define SDHCI_HAVE_DMA 0x01 #define SDHCI_PLATFORM_TRANSFER 0x02 #define SDHCI_NON_REMOVABLE 0x04 +#define SDHCI_TUNING_SUPPORTED 0x08 +#define SDHCI_TUNING_ENABLED 0x10 +#define SDHCI_SDR50_NEEDS_TUNING 0x20 u_char version; int timeout; /* Transfer timeout */ uint32_t max_clk; /* Max possible freq */ @@ -351,14 +355,27 @@ struct sdhci_slot { card_delayed_task;/* Card insert delayed task */ struct callout card_poll_callout;/* Card present polling callout */ struct callout timeout_callout;/* Card command/data response timeout */ + struct callout retune_callout; /* Re-tuning mode 1 callout */ struct mmc_host host; /* Host parameters */ struct mmc_request *req; /* Current request */ struct mmc_command *curcmd; /* Current command of current request */ + struct mmc_request *tune_req; /* Tuning request */ + struct mmc_command *tune_cmd; /* Tuning command of tuning request */ + struct mmc_data *tune_data; /* Tuning data of tuning command */ + uint32_t retune_ticks; /* Re-tuning callout ticks [hz] */ uint32_t intmask; /* Current interrupt mask */ uint32_t clock; /* Current clock freq. */ size_t offset; /* Data buffer offset */ uint8_t hostctrl; /* Current host control register */ + uint8_t retune_count; /* Controller re-tuning count [s] */ + uint8_t retune_mode; /* Controller re-tuning mode */ +#define SDHCI_RETUNE_MODE_1 0x00 +#define SDHCI_RETUNE_MODE_2 0x01 +#define SDHCI_RETUNE_MODE_3 0x02 + uint8_t retune_req; /* Re-tuning request status */ +#define SDHCI_RETUNE_REQ_NEEDED 0x01 /* Re-tuning w/o circuit reset needed */ +#define SDHCI_RETUNE_REQ_RESET 0x02 /* Re-tuning w/ circuit reset needed */ u_char power; /* Current power */ u_char bus_busy; /* Bus busy status */ u_char cmd_done; /* CMD command part done flag */ @@ -368,7 +385,6 @@ struct sdhci_slot { #define STOP_STARTED 2 #define SDHCI_USE_DMA 4 /* Use DMA for this req. */ #define PLATFORM_DATA_STARTED 8 /* Data xfer is handled by platform */ - struct mtx mtx; /* Slot mutex */ #ifdef MMCCAM /* CAM stuff */ @@ -392,7 +408,9 @@ int sdhci_cleanup_slot(struct sdhci_slot *slot); int sdhci_generic_suspend(struct sdhci_slot *slot); int sdhci_generic_resume(struct sdhci_slot *slot); int sdhci_generic_update_ios(device_t brdev, device_t reqdev); +int sdhci_generic_tune(device_t brdev, device_t reqdev, bool hs400); int sdhci_generic_switch_vccq(device_t brdev, device_t reqdev); +int sdhci_generic_retune(device_t brdev, device_t reqdev, bool reset); int sdhci_generic_request(device_t brdev, device_t reqdev, struct mmc_request *req); int sdhci_generic_get_ro(device_t brdev, device_t reqdev); diff --git a/sys/dev/sdhci/sdhci_acpi.c b/sys/dev/sdhci/sdhci_acpi.c index e6c63824baf..5e7e0748ecb 100644 --- a/sys/dev/sdhci/sdhci_acpi.c +++ b/sys/dev/sdhci/sdhci_acpi.c @@ -371,6 +371,8 @@ static device_method_t sdhci_methods[] = { /* mmcbr_if */ DEVMETHOD(mmcbr_update_ios, sdhci_generic_update_ios), DEVMETHOD(mmcbr_switch_vccq, sdhci_generic_switch_vccq), + DEVMETHOD(mmcbr_tune, sdhci_generic_tune), + DEVMETHOD(mmcbr_retune, sdhci_generic_retune), DEVMETHOD(mmcbr_request, sdhci_generic_request), DEVMETHOD(mmcbr_get_ro, sdhci_generic_get_ro), DEVMETHOD(mmcbr_acquire_host, sdhci_generic_acquire_host), diff --git a/sys/dev/sdhci/sdhci_if.m b/sys/dev/sdhci/sdhci_if.m index d77de7746af..b70f04c8d37 100644 --- a/sys/dev/sdhci/sdhci_if.m +++ b/sys/dev/sdhci/sdhci_if.m @@ -58,11 +58,14 @@ # that mmc/sd card drivers call to make requests. # +#include +#include #include -#include #include #include +#include + #include #include diff --git a/sys/dev/sdhci/sdhci_pci.c b/sys/dev/sdhci/sdhci_pci.c index de798cc29e1..fc283d04c65 100644 --- a/sys/dev/sdhci/sdhci_pci.c +++ b/sys/dev/sdhci/sdhci_pci.c @@ -496,6 +496,8 @@ static device_method_t sdhci_methods[] = { /* mmcbr_if */ DEVMETHOD(mmcbr_update_ios, sdhci_generic_update_ios), DEVMETHOD(mmcbr_switch_vccq, sdhci_generic_switch_vccq), + DEVMETHOD(mmcbr_tune, sdhci_generic_tune), + DEVMETHOD(mmcbr_retune, sdhci_generic_retune), DEVMETHOD(mmcbr_request, sdhci_generic_request), DEVMETHOD(mmcbr_get_ro, sdhci_generic_get_ro), DEVMETHOD(mmcbr_acquire_host, sdhci_generic_acquire_host),