diff --git a/sys/dev/mrsas/mrsas.c b/sys/dev/mrsas/mrsas.c index 4bf7ea543dd..d45cb11e67f 100644 --- a/sys/dev/mrsas/mrsas.c +++ b/sys/dev/mrsas/mrsas.c @@ -442,7 +442,10 @@ mrsas_setup_sysctl(struct mrsas_softc *sc) OID_AUTO, "block_sync_cache", CTLFLAG_RW, &sc->block_sync_cache, 0, "Block SYNC CACHE at driver. "); - + SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), + OID_AUTO, "stream detection", CTLFLAG_RW, + &sc->drv_stream_detection, 0, + "Disable/Enable Stream detection. "); } /* @@ -463,6 +466,7 @@ mrsas_get_tunables(struct mrsas_softc *sc) sc->reset_count = 0; sc->reset_in_progress = 0; sc->block_sync_cache = 0; + sc->drv_stream_detection = 1; /* * Grab the global variables. @@ -882,6 +886,7 @@ mrsas_attach(device_t dev) mtx_init(&sc->mpt_cmd_pool_lock, "mrsas_mpt_cmd_pool_lock", NULL, MTX_DEF); mtx_init(&sc->mfi_cmd_pool_lock, "mrsas_mfi_cmd_pool_lock", NULL, MTX_DEF); mtx_init(&sc->raidmap_lock, "mrsas_raidmap_lock", NULL, MTX_DEF); + mtx_init(&sc->stream_lock, "mrsas_stream_lock", NULL, MTX_DEF); /* Intialize linked list */ TAILQ_INIT(&sc->mrsas_mpt_cmd_list_head); @@ -948,6 +953,7 @@ attach_fail_fw: mtx_destroy(&sc->mpt_cmd_pool_lock); mtx_destroy(&sc->mfi_cmd_pool_lock); mtx_destroy(&sc->raidmap_lock); + mtx_destroy(&sc->stream_lock); attach_fail: if (sc->reg_res) { bus_release_resource(sc->mrsas_dev, SYS_RES_MEMORY, @@ -1070,6 +1076,14 @@ mrsas_detach(device_t dev) mrsas_flush_cache(sc); mrsas_shutdown_ctlr(sc, MR_DCMD_CTRL_SHUTDOWN); mrsas_disable_intr(sc); + + if (sc->is_ventura && sc->streamDetectByLD) { + for (i = 0; i < MAX_LOGICAL_DRIVES_EXT; ++i) + free(sc->streamDetectByLD[i], M_MRSAS); + free(sc->streamDetectByLD, M_MRSAS); + sc->streamDetectByLD = NULL; + } + mrsas_cam_detach(sc); mrsas_teardown_intr(sc); mrsas_free_mem(sc); @@ -1081,6 +1095,7 @@ mrsas_detach(device_t dev) mtx_destroy(&sc->mpt_cmd_pool_lock); mtx_destroy(&sc->mfi_cmd_pool_lock); mtx_destroy(&sc->raidmap_lock); + mtx_destroy(&sc->stream_lock); /* Wait for all the semaphores to be released */ while (sema_value(&sc->ioctl_count_sema) != MRSAS_MAX_IOCTL_CMDS) @@ -2166,6 +2181,7 @@ mrsas_init_fw(struct mrsas_softc *sc) u_int32_t scratch_pad_2, scratch_pad_3; int msix_enable = 0; int fw_msix_count = 0; + int i, j; /* Make sure Firmware is ready */ ret = mrsas_transition_to_ready(sc, ocr); @@ -2294,6 +2310,30 @@ mrsas_init_fw(struct mrsas_softc *sc) device_printf(sc->mrsas_dev, "Get LD lsit failed.\n"); return (1); } + + if (sc->is_ventura && sc->drv_stream_detection) { + sc->streamDetectByLD = malloc(sizeof(PTR_LD_STREAM_DETECT) * + MAX_LOGICAL_DRIVES_EXT, M_MRSAS, M_NOWAIT); + if (!sc->streamDetectByLD) { + device_printf(sc->mrsas_dev, + "unable to allocate stream detection for pool of LDs\n"); + return (1); + } + for (i = 0; i < MAX_LOGICAL_DRIVES_EXT; ++i) { + sc->streamDetectByLD[i] = malloc(sizeof(LD_STREAM_DETECT), M_MRSAS, M_NOWAIT); + if (!sc->streamDetectByLD[i]) { + device_printf(sc->mrsas_dev, "unable to allocate stream detect by LD\n"); + for (j = 0; j < i; ++j) + free(sc->streamDetectByLD[j], M_MRSAS); + free(sc->streamDetectByLD, M_MRSAS); + sc->streamDetectByLD = NULL; + return (1); + } + memset(sc->streamDetectByLD[i], 0, sizeof(LD_STREAM_DETECT)); + sc->streamDetectByLD[i]->mruBitMap = MR_STREAM_BITMAP; + } + } + /* * Compute the max allowed sectors per IO: The controller info has * two limits on max sectors. Driver should use the minimum of these @@ -3133,6 +3173,13 @@ mrsas_reset_ctrl(struct mrsas_softc *sc, u_int8_t reset_reason) megasas_setup_jbod_map(sc); + if (sc->is_ventura && sc->streamDetectByLD) { + for (j = 0; j < MAX_LOGICAL_DRIVES_EXT; ++j) { + memset(sc->streamDetectByLD[i], 0, sizeof(LD_STREAM_DETECT)); + sc->streamDetectByLD[i]->mruBitMap = MR_STREAM_BITMAP; + } + } + mrsas_clear_bit(MRSAS_FUSION_IN_RESET, &sc->reset_flags); mrsas_enable_intr(sc); sc->adprecovery = MRSAS_HBA_OPERATIONAL; diff --git a/sys/dev/mrsas/mrsas.h b/sys/dev/mrsas/mrsas.h index 8ad1b4b11de..7f4616c282f 100644 --- a/sys/dev/mrsas/mrsas.h +++ b/sys/dev/mrsas/mrsas.h @@ -121,6 +121,12 @@ __FBSDID("$FreeBSD$"); #define THRESHOLD_REPLY_COUNT 50 #define MAX_MSIX_COUNT 128 +#define MAX_STREAMS_TRACKED 8 +#define MR_STREAM_BITMAP 0x76543210 +#define BITS_PER_INDEX_STREAM 4 /* number of bits per index in U32 TrackStream */ +#define STREAM_MASK ((1 << BITS_PER_INDEX_STREAM) - 1) +#define ZERO_LAST_STREAM 0x0fffffff + /* * Boolean types */ @@ -791,7 +797,8 @@ typedef struct _MR_SPAN_BLOCK_INFO { typedef struct _MR_LD_RAID { struct { u_int32_t fpCapable:1; - u_int32_t reserved5:3; + u_int32_t raCapable:1; + u_int32_t reserved5:2; u_int32_t ldPiMode:4; u_int32_t pdPiMode:4; u_int32_t encryptionType:8; @@ -1011,6 +1018,7 @@ struct IO_REQUEST_INFO { /* span[7:5], arm[4:0] */ u_int8_t span_arm; u_int8_t pd_after_lb; + boolean_t raCapable; }; /* @@ -1033,6 +1041,29 @@ struct MR_PD_CFG_SEQ_NUM_SYNC { struct MR_PD_CFG_SEQ seq[1]; } __packed; +typedef struct _STREAM_DETECT { + u_int64_t nextSeqLBA; + struct megasas_cmd_fusion *first_cmd_fusion; + struct megasas_cmd_fusion *last_cmd_fusion; + u_int32_t countCmdsInStream; + u_int16_t numSGEsInGroup; + u_int8_t isRead; + u_int8_t groupDepth; + boolean_t groupFlush; + u_int8_t reserved[7]; +} STREAM_DETECT, *PTR_STREAM_DETECT; + +typedef struct _LD_STREAM_DETECT { + boolean_t writeBack; + boolean_t FPWriteEnabled; + boolean_t membersSSDs; + boolean_t fpCacheBypassCapable; + u_int32_t mruBitMap; + volatile long iosToFware; + volatile long writeBytesOutstanding; + STREAM_DETECT streamTrack[MAX_STREAMS_TRACKED]; +} LD_STREAM_DETECT, *PTR_LD_STREAM_DETECT; + typedef struct _MR_LD_TARGET_SYNC { u_int8_t targetId; @@ -2950,6 +2981,7 @@ struct mrsas_softc { struct mtx mfi_cmd_pool_lock; struct mtx raidmap_lock; struct mtx aen_lock; + struct mtx stream_lock; struct selinfo mrsas_select; uint32_t mrsas_aen_triggered; uint32_t mrsas_poll_waiting; @@ -3002,6 +3034,7 @@ struct mrsas_softc { u_int32_t reset_in_progress; u_int32_t reset_count; u_int32_t block_sync_cache; + u_int32_t drv_stream_detection; u_int8_t fw_sync_cache_support; mrsas_atomic_t target_reset_outstanding; #define MRSAS_MAX_TM_TARGETS (MRSAS_MAX_PD + MRSAS_MAX_LD_IDS) @@ -3078,6 +3111,7 @@ struct mrsas_softc { /* Non dma-able memory. Driver local copy. */ MR_DRV_RAID_MAP_ALL *ld_drv_map[2]; + PTR_LD_STREAM_DETECT *streamDetectByLD; }; /* Compatibility shims for different OS versions */ diff --git a/sys/dev/mrsas/mrsas_cam.c b/sys/dev/mrsas/mrsas_cam.c index 5b29917ad91..72f904afdac 100644 --- a/sys/dev/mrsas/mrsas_cam.c +++ b/sys/dev/mrsas/mrsas_cam.c @@ -801,6 +801,70 @@ mrsas_build_ldio_rw(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd, return (0); } +/* stream detection on read and and write IOs */ +static void +mrsas_stream_detect(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd, + struct IO_REQUEST_INFO *io_info) +{ + u_int32_t device_id = io_info->ldTgtId; + LD_STREAM_DETECT *current_ld_SD = sc->streamDetectByLD[device_id]; + u_int32_t *track_stream = ¤t_ld_SD->mruBitMap; + u_int32_t streamNum, shiftedValues, unshiftedValues; + u_int32_t indexValueMask, shiftedValuesMask; + int i; + boolean_t isReadAhead = false; + STREAM_DETECT *current_SD; + + /* find possible stream */ + for (i = 0; i < MAX_STREAMS_TRACKED; ++i) { + streamNum = (*track_stream >> (i * BITS_PER_INDEX_STREAM)) & + STREAM_MASK; + current_SD = ¤t_ld_SD->streamTrack[streamNum]; + /* + * if we found a stream, update the raid context and + * also update the mruBitMap + */ + if (current_SD->nextSeqLBA && + io_info->ldStartBlock >= current_SD->nextSeqLBA && + (io_info->ldStartBlock <= (current_SD->nextSeqLBA+32)) && + (current_SD->isRead == io_info->isRead)) { + if (io_info->ldStartBlock != current_SD->nextSeqLBA && + (!io_info->isRead || !isReadAhead)) { + /* + * Once the API availible we need to change this. + * At this point we are not allowing any gap + */ + continue; + } + cmd->io_request->RaidContext.raid_context_g35.streamDetected = TRUE; + current_SD->nextSeqLBA = io_info->ldStartBlock + io_info->numBlocks; + /* + * update the mruBitMap LRU + */ + shiftedValuesMask = (1 << i * BITS_PER_INDEX_STREAM) - 1 ; + shiftedValues = ((*track_stream & shiftedValuesMask) << + BITS_PER_INDEX_STREAM); + indexValueMask = STREAM_MASK << i * BITS_PER_INDEX_STREAM; + unshiftedValues = (*track_stream) & + (~(shiftedValuesMask | indexValueMask)); + *track_stream = + (unshiftedValues | shiftedValues | streamNum); + return; + } + } + /* + * if we did not find any stream, create a new one from the least recently used + */ + streamNum = (*track_stream >> + ((MAX_STREAMS_TRACKED - 1) * BITS_PER_INDEX_STREAM)) & STREAM_MASK; + current_SD = ¤t_ld_SD->streamTrack[streamNum]; + current_SD->isRead = io_info->isRead; + current_SD->nextSeqLBA = io_info->ldStartBlock + io_info->numBlocks; + *track_stream = (((*track_stream & ZERO_LAST_STREAM) << 4) | streamNum); + return; +} + + /* * mrsas_setup_io: Set up data including Fast Path I/O * input: Adapter instance soft state @@ -916,6 +980,15 @@ mrsas_setup_io(struct mrsas_softc *sc, struct mrsas_mpt_cmd *cmd, cmd->request_desc->SCSIIO.MSIxIndex = sc->msix_vectors ? smp_processor_id() % sc->msix_vectors : 0; + if (sc->is_ventura && sc->streamDetectByLD) { + mtx_lock(&sc->stream_lock); + mrsas_stream_detect(sc, cmd, &io_info); + mtx_unlock(&sc->stream_lock); + /* In ventura if stream detected for a read and it is read ahead capable make this IO as LDIO */ + if (io_request->RaidContext.raid_context_g35.streamDetected && + io_info.isRead && io_info.raCapable) + fp_possible = FALSE; + } if (fp_possible) { mrsas_set_pd_lba(io_request, csio->cdb_len, &io_info, ccb, map_ptr, diff --git a/sys/dev/mrsas/mrsas_fp.c b/sys/dev/mrsas/mrsas_fp.c index 1834ecfa1e3..4d9862233d6 100644 --- a/sys/dev/mrsas/mrsas_fp.c +++ b/sys/dev/mrsas/mrsas_fp.c @@ -1014,6 +1014,9 @@ MR_BuildRaidContext(struct mrsas_softc *sc, struct IO_REQUEST_INFO *io_info, ld = MR_TargetIdToLdGet(ldTgtId, map); raid = MR_LdRaidGet(ld, map); + /* check read ahead bit */ + io_info->raCapable = raid->capability.raCapable; + if (raid->rowDataSize == 0) { if (MR_LdSpanPtrGet(ld, 0, map)->spanRowDataSize == 0) return FALSE;