diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h index 2655dae1c0f..f742875dd47 100644 --- a/sys/dev/cxgbe/adapter.h +++ b/sys/dev/cxgbe/adapter.h @@ -750,7 +750,7 @@ struct adapter { void *tom_softc; /* (struct tom_data *) */ struct tom_tunables tt; void *iwarp_softc; /* (struct c4iw_dev *) */ - void *iscsi_softc; + void *iscsi_softc; /* (struct cxgbei_data *) */ #endif struct l2t_data *l2t; /* L2 table */ struct tid_info tids; diff --git a/sys/dev/cxgbe/cxgbei/cxgbei.c b/sys/dev/cxgbe/cxgbei/cxgbei.c index 1fa68077ed7..7f397541808 100644 --- a/sys/dev/cxgbe/cxgbei/cxgbei.c +++ b/sys/dev/cxgbe/cxgbei/cxgbei.c @@ -135,17 +135,10 @@ find_ulp_mbuf_cb(struct mbuf *m) * The location of the pagepod entry is encoded into ddp tag which is used as * the base for ITT/TTT. */ -#define T4_DDP -#ifdef T4_DDP + /* * functions to program the pagepod in h/w */ -static void * -t4_tdev2ddp(void *tdev) -{ - struct adapter *sc = ((struct toedev *)tdev)->tod_softc; - return (sc->iscsi_softc); -} static void inline ppod_set(struct pagepod *ppod, struct cxgbei_ulp2_pagepod_hdr *hdr, @@ -193,27 +186,27 @@ ulp_mem_io_set_hdr(struct adapter *sc, int tid, struct ulp_mem_io *req, #define PCIE_MEMWIN_MAX_NPPODS 16 /* 1024/PPOD_SIZE */ static int -ppod_write_idata(struct cxgbei_ulp2_ddp_info *ddp, +ppod_write_idata(struct cxgbei_data *ci, struct cxgbei_ulp2_pagepod_hdr *hdr, unsigned int idx, unsigned int npods, struct cxgbei_ulp2_gather_list *gl, unsigned int gl_pidx, struct toepcb *toep) { - unsigned int dlen = PPOD_SIZE * npods; - unsigned int pm_addr = idx * PPOD_SIZE + ddp->llimit; - unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) + - sizeof(struct ulptx_idata) + dlen, 16); + u_int dlen = PPOD_SIZE * npods; + u_int pm_addr = idx * PPOD_SIZE + ci->llimit; + u_int wr_len = roundup(sizeof(struct ulp_mem_io) + + sizeof(struct ulptx_idata) + dlen, 16); struct ulp_mem_io *req; struct ulptx_idata *idata; struct pagepod *ppod; - unsigned int i; + u_int i; struct wrqe *wr; struct adapter *sc = toep->port->adapter; wr = alloc_wrqe(wr_len, toep->ctrlq); if (wr == NULL) { - printf("%s: alloc wrqe failed\n", __func__); - return ENOMEM; + CXGBE_UNIMPLEMENTED("ppod_write_idata: alloc_wrqe failure"); + return (ENOMEM); } req = wrtod(wr); @@ -233,25 +226,16 @@ ppod_write_idata(struct cxgbei_ulp2_ddp_info *ddp, return 0; } -static int -t4_ddp_set_map(struct cxgbei_ulp2_ddp_info *ddp, - void *isockp, struct cxgbei_ulp2_pagepod_hdr *hdr, - unsigned int idx, unsigned int npods, - struct cxgbei_ulp2_gather_list *gl, int reply) +int +t4_ddp_set_map(struct cxgbei_data *ci, void *isockp, + struct cxgbei_ulp2_pagepod_hdr *hdr, u_int idx, u_int npods, + struct cxgbei_ulp2_gather_list *gl, int reply) { - iscsi_socket *isock = (iscsi_socket *)isockp; - struct socket *sk; - struct toepcb *toep; - struct tcpcb *tp; + struct iscsi_socket *isock = (struct iscsi_socket *)isockp; + struct toepcb *toep = isock->toep; int err; unsigned int pidx = 0, w_npods = 0, cnt; - if (isock == NULL) - return EINVAL; - sk = isock->sock; - tp = so_sototcpcb(sk); - toep = tp->t_toe; - /* * on T4, if we use a mix of IMMD and DSGL with ULP_MEM_WRITE, * the order would not be garanteed, so we will stick with IMMD @@ -266,7 +250,7 @@ t4_ddp_set_map(struct cxgbei_ulp2_ddp_info *ddp, cnt = npods - w_npods; if (cnt > ULPMEM_IDATA_MAX_NPPODS) cnt = ULPMEM_IDATA_MAX_NPPODS; - err = ppod_write_idata(ddp, hdr, idx, cnt, gl, + err = ppod_write_idata(ci, hdr, idx, cnt, gl, pidx, toep); if (err) { printf("%s: ppod_write_idata failed\n", __func__); @@ -276,126 +260,29 @@ t4_ddp_set_map(struct cxgbei_ulp2_ddp_info *ddp, return err; } -static void -t4_ddp_clear_map(struct cxgbei_ulp2_ddp_info *ddp, - struct cxgbei_ulp2_gather_list *gl, - unsigned int tag, unsigned int idx, unsigned int npods, - iscsi_socket *isock) +void +t4_ddp_clear_map(struct cxgbei_data *ci, struct cxgbei_ulp2_gather_list *gl, + u_int tag, u_int idx, u_int npods, struct iscsi_socket *isock) { - struct socket *sk; - struct toepcb *toep; - struct tcpcb *tp; + struct toepcb *toep = isock->toep; int err = -1; - - sk = isock->sock; - tp = so_sototcpcb(sk); - toep = tp->t_toe; - - /* send via immediate data */ - unsigned int pidx = 0; - unsigned int w_npods = 0; - unsigned int cnt; + u_int pidx = 0; + u_int w_npods = 0; + u_int cnt; for (; w_npods < npods; idx += cnt, w_npods += cnt, pidx += PPOD_PAGES) { cnt = npods - w_npods; if (cnt > ULPMEM_IDATA_MAX_NPPODS) cnt = ULPMEM_IDATA_MAX_NPPODS; - err = ppod_write_idata(ddp, NULL, idx, cnt, gl, 0, toep); + err = ppod_write_idata(ci, NULL, idx, cnt, gl, 0, toep); if (err) break; } } -#endif - -/* - * cxgbei device management - * maintains a list of the cxgbei devices - */ -typedef struct offload_device { - SLIST_ENTRY(offload_device) link; - unsigned char d_version; - unsigned char d_tx_hdrlen; /* CPL_TX_DATA, < 256 */ - unsigned char d_ulp_rx_datagap; /* for coalesced iscsi msg */ - unsigned char filler; - - unsigned int d_flag; - unsigned int d_payload_tmax; - unsigned int d_payload_rmax; - - struct cxgbei_ulp2_tag_format d_tag_format; - void *d_tdev; - void *d_pdev; - void* (*tdev2ddp)(void *tdev); -}offload_device; - -SLIST_HEAD(, offload_device) odev_list; - -static void t4_unregister_cpl_handler_with_tom(struct adapter *sc); -static offload_device * -offload_device_new(void *tdev) -{ - offload_device *odev = NULL; - odev = malloc(sizeof(struct offload_device), - M_CXGBE, M_NOWAIT | M_ZERO); - if (odev) { - odev->d_tdev = tdev; - SLIST_INSERT_HEAD(&odev_list, odev, link); - } - - return odev; -} - -static offload_device * -offload_device_find(struct toedev *tdev) -{ - offload_device *odev = NULL; - - if (!SLIST_EMPTY(&odev_list)) { - SLIST_FOREACH(odev, &odev_list, link) { - if (odev->d_tdev == tdev) - break; - } - } - return odev; -} - -static void -cxgbei_odev_cleanup(offload_device *odev) -{ - struct toedev *tdev = odev->d_tdev; - struct adapter *sc = (struct adapter *)tdev->tod_softc; - - /* de-register ULP CPL handlers with TOM */ - t4_unregister_cpl_handler_with_tom(sc); - if (odev->d_flag & ODEV_FLAG_ULP_DDP_ENABLED) { - if (sc->iscsi_softc) - cxgbei_ulp2_ddp_cleanup( - (struct cxgbei_ulp2_ddp_info **)&sc->iscsi_softc); - } - return; -} - -static void -offload_device_remove() -{ - offload_device *odev = NULL, *next = NULL; - - if (SLIST_EMPTY(&odev_list)) - return; - - for (odev = SLIST_FIRST(&odev_list); odev != NULL; odev = next) { - SLIST_REMOVE(&odev_list, odev, offload_device, link); - next = SLIST_NEXT(odev, link); - cxgbei_odev_cleanup(odev); - free(odev, M_CXGBE); - } - - return; -} static int -cxgbei_map_sg(cxgbei_sgl *sgl, struct ccb_scsiio *csio) +cxgbei_map_sg(struct cxgbei_sgl *sgl, struct ccb_scsiio *csio) { unsigned int data_len = csio->dxfer_len; unsigned int sgoffset = (uint64_t)csio->data_ptr & PAGE_MASK; @@ -430,7 +317,7 @@ cxgbei_map_sg(cxgbei_sgl *sgl, struct ccb_scsiio *csio) } static int -cxgbei_map_sg_tgt(cxgbei_sgl *sgl, union ctl_io *io) +cxgbei_map_sg_tgt(struct cxgbei_sgl *sgl, union ctl_io *io) { unsigned int data_len, sgoffset, nsge; unsigned char *sgaddr; @@ -486,34 +373,19 @@ cxgbei_map_sg_tgt(cxgbei_sgl *sgl, union ctl_io *io) } static int -t4_sk_ddp_tag_reserve(iscsi_socket *isock, unsigned int xferlen, - cxgbei_sgl *sgl, unsigned int sgcnt, - unsigned int *ddp_tag) +t4_sk_ddp_tag_reserve(struct cxgbei_data *ci, struct iscsi_socket *isock, + u_int xferlen, struct cxgbei_sgl *sgl, u_int sgcnt, u_int *ddp_tag) { - offload_device *odev = isock->s_odev; - struct toedev *tdev = odev->d_tdev; struct cxgbei_ulp2_gather_list *gl; int err = -EINVAL; - struct adapter *sc = tdev->tod_softc; - struct cxgbei_ulp2_ddp_info *ddp; + struct toepcb *toep = isock->toep; - ddp = (struct cxgbei_ulp2_ddp_info *)sc->iscsi_softc; - if (ddp == NULL) - return ENOMEM; - - gl = cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(xferlen, sgl, sgcnt, - odev->d_tdev, 0); + gl = cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(xferlen, sgl, sgcnt, ci, 0); if (gl) { - err = cxgbei_ulp2_ddp_tag_reserve(odev->tdev2ddp(tdev), - isock, - isock->s_tid, - &odev->d_tag_format, - ddp_tag, gl, - 0, 0); + err = cxgbei_ulp2_ddp_tag_reserve(ci, isock, toep->tid, + &ci->tag_format, ddp_tag, gl, 0, 0); if (err) { - CTR1(KTR_CXGBE, - "%s: ddp_tag_reserve failed\n", __func__); - cxgbei_ulp2_ddp_release_gl(gl, odev->d_tdev); + cxgbei_ulp2_ddp_release_gl(ci, gl); } } @@ -525,17 +397,17 @@ cxgbei_task_reserve_itt(struct icl_conn *ic, void **prv, struct ccb_scsiio *scmd, unsigned int *itt) { int xferlen = scmd->dxfer_len; - cxgbei_task_data *tdata = NULL; - cxgbei_sgl *sge = NULL; - struct socket *so = ic->ic_socket; - iscsi_socket *isock = (iscsi_socket *)(so)->so_emuldata; + struct cxgbei_task_data *tdata = NULL; + struct cxgbei_sgl *sge = NULL; + struct iscsi_socket *isock = ic->ic_ofld_prv0; + struct toepcb *toep = isock->toep; + struct adapter *sc = td_adapter(toep->td); + struct cxgbei_data *ci = sc->iscsi_softc; int err = -1; - offload_device *odev = isock->s_odev; - tdata = (cxgbei_task_data *)*prv; - if ((xferlen == 0) || (tdata == NULL)) { + tdata = (struct cxgbei_task_data *)*prv; + if (xferlen == 0 || tdata == NULL) goto out; - } if (xferlen < DDP_THRESHOLD) goto out; @@ -551,10 +423,10 @@ cxgbei_task_reserve_itt(struct icl_conn *ic, void **prv, CTR3(KTR_CXGBE, "%s: *itt:0x%x sc_ddp_tag:0x%x\n", __func__, *itt, tdata->sc_ddp_tag); - if (cxgbei_ulp2_sw_tag_usable(&odev->d_tag_format, + if (cxgbei_ulp2_sw_tag_usable(&ci->tag_format, tdata->sc_ddp_tag)) { - err = t4_sk_ddp_tag_reserve(isock, scmd->dxfer_len, sge, - tdata->nsge, &tdata->sc_ddp_tag); + err = t4_sk_ddp_tag_reserve(ci, isock, scmd->dxfer_len, + sge, tdata->nsge, &tdata->sc_ddp_tag); } else { CTR3(KTR_CXGBE, "%s: itt:0x%x sc_ddp_tag:0x%x not usable\n", @@ -564,7 +436,7 @@ cxgbei_task_reserve_itt(struct icl_conn *ic, void **prv, out: if (err < 0) tdata->sc_ddp_tag = - cxgbei_ulp2_set_non_ddp_tag(&odev->d_tag_format, *itt); + cxgbei_ulp2_set_non_ddp_tag(&ci->tag_format, *itt); return tdata->sc_ddp_tag; } @@ -573,15 +445,16 @@ static unsigned int cxgbei_task_reserve_ttt(struct icl_conn *ic, void **prv, union ctl_io *io, unsigned int *ttt) { - struct socket *so = ic->ic_socket; - iscsi_socket *isock = (iscsi_socket *)(so)->so_emuldata; - cxgbei_task_data *tdata = NULL; - offload_device *odev = isock->s_odev; + struct iscsi_socket *isock = ic->ic_ofld_prv0; + struct toepcb *toep = isock->toep; + struct adapter *sc = td_adapter(toep->td); + struct cxgbei_data *ci = sc->iscsi_softc; + struct cxgbei_task_data *tdata = NULL; int xferlen, err = -1; - cxgbei_sgl *sge = NULL; + struct cxgbei_sgl *sge = NULL; xferlen = (io->scsiio.kern_data_len - io->scsiio.ext_data_filled); - tdata = (cxgbei_task_data *)*prv; + tdata = (struct cxgbei_task_data *)*prv; if ((xferlen == 0) || (tdata == NULL)) goto out; if (xferlen < DDP_THRESHOLD) @@ -594,9 +467,9 @@ cxgbei_task_reserve_ttt(struct icl_conn *ic, void **prv, union ctl_io *io, sge = tdata->sgl; tdata->sc_ddp_tag = *ttt; - if (cxgbei_ulp2_sw_tag_usable(&odev->d_tag_format, tdata->sc_ddp_tag)) { - err = t4_sk_ddp_tag_reserve(isock, xferlen, sge, tdata->nsge, - &tdata->sc_ddp_tag); + if (cxgbei_ulp2_sw_tag_usable(&ci->tag_format, tdata->sc_ddp_tag)) { + err = t4_sk_ddp_tag_reserve(ci, isock, xferlen, sge, + tdata->nsge, &tdata->sc_ddp_tag); } else { CTR2(KTR_CXGBE, "%s: sc_ddp_tag:0x%x not usable\n", __func__, tdata->sc_ddp_tag); @@ -604,102 +477,95 @@ cxgbei_task_reserve_ttt(struct icl_conn *ic, void **prv, union ctl_io *io, out: if (err < 0) tdata->sc_ddp_tag = - cxgbei_ulp2_set_non_ddp_tag(&odev->d_tag_format, *ttt); + cxgbei_ulp2_set_non_ddp_tag(&ci->tag_format, *ttt); return tdata->sc_ddp_tag; } static int -t4_sk_ddp_tag_release(iscsi_socket *isock, unsigned int ddp_tag) +t4_sk_ddp_tag_release(struct iscsi_socket *isock, unsigned int ddp_tag) { - offload_device *odev = isock->s_odev; - struct toedev *tdev = odev->d_tdev; + struct toepcb *toep = isock->toep; + struct adapter *sc = td_adapter(toep->td); + struct cxgbei_data *ci = sc->iscsi_softc; - cxgbei_ulp2_ddp_tag_release(odev->tdev2ddp(tdev), ddp_tag, isock); - return 0; + cxgbei_ulp2_ddp_tag_release(ci, ddp_tag, isock); + + return (0); } -#ifdef T4_DDP -static struct cxgbei_ulp2_ddp_info * -t4_ddp_init(struct ifnet *dev, struct toedev *tdev) + +static int +cxgbei_ddp_init(struct adapter *sc, struct cxgbei_data *ci) { - struct cxgbei_ulp2_ddp_info *ddp; - struct adapter *sc = tdev->tod_softc; - struct ulp_iscsi_info uinfo; + int nppods, bits, max_sz, rc; + static const u_int pgsz_order[] = {0, 1, 2, 3}; - memset(&uinfo, 0, sizeof(struct ulp_iscsi_info)); - uinfo.llimit = sc->vres.iscsi.start; - uinfo.ulimit = sc->vres.iscsi.start + sc->vres.iscsi.size - 1; - uinfo.max_rxsz = uinfo.max_txsz = - G_MAXRXDATA(t4_read_reg(sc, A_TP_PARA_REG2)); + MPASS(sc->vres.iscsi.size > 0); - if (sc->vres.iscsi.size == 0) { - printf("%s: iSCSI capabilities not enabled.\n", __func__); - return NULL; + ci->llimit = sc->vres.iscsi.start; + ci->ulimit = sc->vres.iscsi.start + sc->vres.iscsi.size - 1; + max_sz = G_MAXRXDATA(t4_read_reg(sc, A_TP_PARA_REG2)); + + nppods = sc->vres.iscsi.size >> IPPOD_SIZE_SHIFT; + if (nppods <= 1024) + return (ENXIO); + + bits = fls(nppods); + if (bits > IPPOD_IDX_MAX_SIZE) + bits = IPPOD_IDX_MAX_SIZE; + nppods = (1 << (bits - 1)) - 1; + + rc = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR, + BUS_SPACE_MAXADDR, NULL, NULL, UINT32_MAX , 8, BUS_SPACE_MAXSIZE, + BUS_DMA_ALLOCNOW, NULL, NULL, &ci->ulp_ddp_tag); + if (rc != 0) { + device_printf(sc->dev, "%s: failed to create DMA tag: %u.\n", + __func__, rc); + return (rc); } - printf("T4, ddp 0x%x ~ 0x%x, size %u, iolen %u, ulpddp:0x%p\n", - uinfo.llimit, uinfo.ulimit, sc->vres.iscsi.size, - uinfo.max_rxsz, sc->iscsi_softc); - cxgbei_ulp2_ddp_init((void *)tdev, - (struct cxgbei_ulp2_ddp_info **)&sc->iscsi_softc, - &uinfo); - ddp = (struct cxgbei_ulp2_ddp_info *)sc->iscsi_softc; - if (ddp) { - unsigned int pgsz_order[4]; - int i; - - for (i = 0; i < 4; i++) - pgsz_order[i] = uinfo.pgsz_factor[i]; - - t4_iscsi_init(dev, uinfo.tagmask, pgsz_order); - - ddp->ddp_set_map = t4_ddp_set_map; - ddp->ddp_clear_map = t4_ddp_clear_map; + ci->colors = malloc(nppods * sizeof(char), M_CXGBE, M_NOWAIT | M_ZERO); + ci->gl_map = malloc(nppods * sizeof(struct cxgbei_ulp2_gather_list *), + M_CXGBE, M_NOWAIT | M_ZERO); + if (ci->colors == NULL || ci->gl_map == NULL) { + bus_dma_tag_destroy(ci->ulp_ddp_tag); + free(ci->colors, M_CXGBE); + free(ci->gl_map, M_CXGBE); + return (ENOMEM); } - return ddp; -} -#endif -static struct socket * -cpl_find_sock(struct adapter *sc, unsigned int hwtid) -{ - struct socket *sk; - struct toepcb *toep = lookup_tid(sc, hwtid); - struct inpcb *inp = toep->inp; + mtx_init(&ci->map_lock, "ddp lock", NULL, MTX_DEF | MTX_DUPOK); + ci->max_txsz = ci->max_rxsz = min(max_sz, ULP2_MAX_PKT_SIZE); + ci->nppods = nppods; + ci->idx_last = nppods; + ci->idx_bits = bits; + ci->idx_mask = (1 << bits) - 1; + ci->rsvd_tag_mask = (1 << (bits + IPPOD_IDX_SHIFT)) - 1; - INP_WLOCK(inp); - sk = inp->inp_socket; - INP_WUNLOCK(inp); - if (sk == NULL) - CTR2(KTR_CXGBE, - "%s: T4 CPL tid 0x%x, sk NULL.\n", __func__, hwtid); - return sk; + ci->tag_format.sw_bits = bits; + ci->tag_format.rsvd_bits = bits; + ci->tag_format.rsvd_shift = IPPOD_IDX_SHIFT; + ci->tag_format.rsvd_mask = ci->idx_mask; + + t4_iscsi_init(sc, ci->idx_mask << IPPOD_IDX_SHIFT, pgsz_order); + + return (rc); } static void -process_rx_iscsi_hdr(struct socket *sk, struct mbuf *m) +process_rx_iscsi_hdr(struct toepcb *toep, struct mbuf *m) { - struct tcpcb *tp = so_sototcpcb(sk); - struct toepcb *toep = tp->t_toe; - struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *); struct ulp_mbuf_cb *cb, *lcb; struct mbuf *lmbuf; - unsigned char *byte; - iscsi_socket *isock = (iscsi_socket *)(sk)->so_emuldata; - unsigned int hlen, dlen, plen; + u_char *byte; + struct iscsi_socket *isock = toep->ulpcb; + struct tcpcb *tp = intotcpcb(toep->inp); + u_int hlen, dlen, plen; - if (isock == NULL) - goto err_out; + MPASS(isock != NULL); + M_ASSERTPKTHDR(m); - if (toep == NULL) - goto err_out; - if ((m->m_flags & M_PKTHDR) == 0) { - printf("%s: m:%p no M_PKTHDR can't allocate m_tag\n", - __func__, m); - goto err_out; - } - - mtx_lock(&isock->iscsi_rcv_mbufq.lock); + mtx_lock(&isock->iscsi_rcvq_lock); /* allocate m_tag to hold ulp info */ cb = get_ulp_mbuf_cb(m); @@ -715,7 +581,6 @@ process_rx_iscsi_hdr(struct socket *sk, struct mbuf *m) /* figure out if this is the pdu header or data */ cb->ulp_mode = ULP_MODE_ISCSI; if (isock->mbuf_ulp_lhdr == NULL) { - iscsi_socket *isock = (iscsi_socket *)(sk)->so_emuldata; isock->mbuf_ulp_lhdr = lmbuf = m; lcb = cb; @@ -761,15 +626,13 @@ process_rx_iscsi_hdr(struct socket *sk, struct mbuf *m) m->m_len += 4 - (m->m_len % 4); } } - mbufq_tail(&isock->iscsi_rcv_mbufq, m); - mtx_unlock(&isock->iscsi_rcv_mbufq.lock); + mbufq_enqueue(&isock->iscsi_rcvq, m); + mtx_unlock(&isock->iscsi_rcvq_lock); return; err_out1: - mtx_unlock(&isock->iscsi_rcv_mbufq.lock); -err_out: + mtx_unlock(&isock->iscsi_rcvq_lock); m_freem(m); - return; } /* hand over received PDU to iscsi_initiator */ @@ -787,7 +650,7 @@ iscsi_conn_receive_pdu(struct iscsi_socket *isock) panic("%s: failed to alloc icl_pdu\n", __func__); return; } - m = mbufq_peek(&isock->iscsi_rcv_mbufq); + m = mbufq_first(&isock->iscsi_rcvq); if (m) { cb = find_ulp_mbuf_cb(m); if (cb == NULL) { @@ -798,22 +661,22 @@ iscsi_conn_receive_pdu(struct iscsi_socket *isock) goto err_out; } /* BHS */ - mbufq_dequeue(&isock->iscsi_rcv_mbufq); + mbufq_dequeue(&isock->iscsi_rcvq); data_len = cb->ulp.iscsi.pdulen; - CTR5(KTR_CXGBE, "%s: response:%p m:%p m_len:%d data_len:%d\n", + CTR5(KTR_CXGBE, "%s: response:%p m:%p m_len:%d data_len:%d", __func__, response, m, m->m_len, data_len); response->ip_bhs_mbuf = m; response->ip_bhs = mtod(response->ip_bhs_mbuf, struct iscsi_bhs *); /* data */ if (cb->flags & SBUF_ULP_FLAG_DATA_RCVD) { - m = mbufq_peek(&isock->iscsi_rcv_mbufq); + m = mbufq_first(&isock->iscsi_rcvq); if (m == NULL) { CTR1(KTR_CXGBE, "%s:No Data\n", __func__); goto err_out; } - mbufq_dequeue(&isock->iscsi_rcv_mbufq); + mbufq_dequeue(&isock->iscsi_rcvq); response->ip_data_mbuf = m; response->ip_data_len += response->ip_data_mbuf->m_len; } else { @@ -829,26 +692,22 @@ err_out: } static void -process_rx_data_ddp(struct socket *sk, void *m) +process_rx_data_ddp(struct toepcb *toep, const struct cpl_rx_data_ddp *cpl) { - struct cpl_rx_data_ddp *cpl = (struct cpl_rx_data_ddp *)m; - struct tcpcb *tp = so_sototcpcb(sk); - struct toepcb *toep = tp->t_toe; - struct inpcb *inp = toep->inp; struct mbuf *lmbuf; struct ulp_mbuf_cb *lcb, *lcb1; unsigned int val, pdulen; - iscsi_socket *isock = (iscsi_socket *)(sk)->so_emuldata; + struct iscsi_socket *isock = toep->ulpcb; + struct inpcb *inp = toep->inp; - if (isock == NULL) - return; + MPASS(isock != NULL); if (isock->mbuf_ulp_lhdr == NULL) { panic("%s: tid 0x%x, rcv RX_DATA_DDP w/o pdu header.\n", - __func__, toep->tid); + __func__, toep->tid); return; } - mtx_lock(&isock->iscsi_rcv_mbufq.lock); + mtx_lock(&isock->iscsi_rcvq_lock); lmbuf = isock->mbuf_ulp_lhdr; if (lmbuf->m_nextpkt) { lcb1 = find_ulp_mbuf_cb(lmbuf->m_nextpkt); @@ -857,7 +716,7 @@ process_rx_data_ddp(struct socket *sk, void *m) lcb = find_ulp_mbuf_cb(isock->mbuf_ulp_lhdr); if (lcb == NULL) { CTR2(KTR_CXGBE, "%s: mtag NULL lmbuf :%p\n", __func__, lmbuf); - mtx_unlock(&isock->iscsi_rcv_mbufq.lock); + mtx_unlock(&isock->iscsi_rcvq_lock); return; } lcb->flags |= SBUF_ULP_FLAG_STATUS_RCVD; @@ -914,251 +773,143 @@ process_rx_data_ddp(struct socket *sk, void *m) #endif iscsi_conn_receive_pdu(isock); - mtx_unlock(&isock->iscsi_rcv_mbufq.lock); + mtx_unlock(&isock->iscsi_rcvq_lock); /* update rx credits */ INP_WLOCK(inp); - SOCK_LOCK(sk); + /* XXXNP: does this want the so_rcv lock? (happens to be the same) */ + SOCK_LOCK(inp->inp_socket); toep->sb_cc += pdulen; - SOCK_UNLOCK(sk); - CTR4(KTR_CXGBE, "sk:%p sb_cc 0x%x, rcv_nxt 0x%x rcv_wnd:0x%lx.\n", - sk, toep->sb_cc, tp->rcv_nxt, tp->rcv_wnd); - t4_rcvd(&toep->td->tod, tp); + SOCK_UNLOCK(inp->inp_socket); + t4_rcvd(&toep->td->tod, intotcpcb(inp)); INP_WUNLOCK(inp); return; } static void -drop_fw_acked_ulp_data(struct socket *sk, struct toepcb *toep, int len) +drop_fw_acked_ulp_data(struct toepcb *toep, int len) { struct mbuf *m, *next; struct ulp_mbuf_cb *cb; - iscsi_socket *isock = (iscsi_socket *)(sk)->so_emuldata; struct icl_pdu *req; + struct iscsi_socket *isock = toep->ulpcb; - if (len == 0 || (isock == NULL)) - return; + MPASS(len > 0); - mtx_lock(&isock->ulp2_wrq.lock); + mtx_lock(&isock->ulp2_wrq_lock); while (len > 0) { m = mbufq_dequeue(&isock->ulp2_wrq); - if(m == NULL) break; + MPASS(m != NULL); /* excess credits */ - for(next = m; next !=NULL; next = next->m_next) + for (next = m; next != NULL; next = next->m_next) { + MPASS(len >= next->m_len); /* excess credits */ len -= next->m_len; + } cb = find_ulp_mbuf_cb(m); - - if (cb && isock && cb->pdu) { + if (cb && cb->pdu) { req = (struct icl_pdu *)cb->pdu; req->ip_bhs_mbuf = NULL; icl_pdu_free(req); } m_freem(m); } - mtx_unlock(&isock->ulp2_wrq.lock); - return; -} - -static void -process_fw4_ack(struct socket *sk, int *plen) -{ - struct tcpcb *tp = so_sototcpcb(sk); - struct toepcb *toep = tp->t_toe; - - drop_fw_acked_ulp_data(sk, toep, *plen); - - return; -} - -static int -do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) -{ - return 0; + mtx_unlock(&isock->ulp2_wrq_lock); } static int do_rx_iscsi_hdr(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { - struct socket *sk; struct adapter *sc = iq->adapter; - struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *); - sk = cpl_find_sock(sc, GET_TID(cpl)); + struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *); /* XXXNP */ + u_int tid = GET_TID(cpl); + struct toepcb *toep = lookup_tid(sc, tid); - if (sk == NULL) - return CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE; + process_rx_iscsi_hdr(toep, m); - process_rx_iscsi_hdr(sk, m); - return 0; -} - -static int -do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) -{ - return 0; + return (0); } static int do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { - struct socket *sk; - struct adapter *sc; - const struct cpl_rx_iscsi_ddp *cpl = (const void *)(rss + 1); + struct adapter *sc = iq->adapter; + const struct cpl_rx_data_ddp *cpl = (const void *)(rss + 1); + u_int tid = GET_TID(cpl); + struct toepcb *toep = lookup_tid(sc, tid); - if (iq == NULL) - return 0; - sc = iq->adapter; - if (sc == NULL) - return 0; + process_rx_data_ddp(toep, cpl); - sk = cpl_find_sock(sc, GET_TID(cpl)); - if (sk == NULL) - return CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE; - - process_rx_data_ddp(sk, (void *)cpl); - return 0; + return (0); } -static int -t4_ulp_mbuf_push(struct socket *so, struct mbuf *m) -{ - struct tcpcb *tp = so_sototcpcb(so); - struct toepcb *toep = tp->t_toe; - struct inpcb *inp = so_sotoinpcb(so); - iscsi_socket *isock = (iscsi_socket *)(so)->so_emuldata;; - if (isock == NULL) { - m_freem(m); - return EINVAL; - } +static int +t4_ulp_mbuf_push(struct iscsi_socket *isock, struct mbuf *m) +{ + struct toepcb *toep = isock->toep; /* append mbuf to ULP queue */ - mtx_lock(&isock->ulp2_writeq.lock); - mbufq_tail(&isock->ulp2_writeq, m); - mtx_unlock(&isock->ulp2_writeq.lock); + mtx_lock(&isock->ulp2_writeq_lock); + mbufq_enqueue(&isock->ulp2_writeq, m); + mtx_unlock(&isock->ulp2_writeq_lock); - INP_WLOCK(inp); + INP_WLOCK(toep->inp); t4_ulp_push_frames(toep->td->tod.tod_softc, toep, 0); - INP_WUNLOCK(inp); - return 0; + INP_WUNLOCK(toep->inp); + + return (0); } static struct mbuf * -iscsi_queue_handler_callback(struct socket *sk, unsigned int cmd, int *qlen) +get_writeq_len(struct toepcb *toep, int *qlen) { - iscsi_socket *isock; - struct mbuf *m0 = NULL; + struct iscsi_socket *isock = toep->ulpcb; - if (sk == NULL) - return NULL; - isock = (iscsi_socket *)(sk)->so_emuldata; - if (isock == NULL) - return NULL; - - switch (cmd) { - case 0:/* PEEK */ - m0 = mbufq_peek(&isock->ulp2_writeq); - break; - case 1:/* QUEUE_LEN */ - *qlen = mbufq_len(&isock->ulp2_writeq); - m0 = mbufq_peek(&isock->ulp2_writeq); - break; - case 2:/* DEQUEUE */ - mtx_lock(&isock->ulp2_writeq.lock); - m0 = mbufq_dequeue(&isock->ulp2_writeq); - mtx_unlock(&isock->ulp2_writeq.lock); - - mtx_lock(&isock->ulp2_wrq.lock); - mbufq_tail(&isock->ulp2_wrq, m0); - mtx_unlock(&isock->ulp2_wrq.lock); - - m0 = mbufq_peek(&isock->ulp2_writeq); - break; - } - return m0; + *qlen = mbufq_len(&isock->ulp2_writeq); + return (mbufq_first(&isock->ulp2_writeq)); } -static void -iscsi_cpl_handler_callback(struct tom_data *td, struct socket *sk, - void *m, unsigned int op) +static struct mbuf * +do_writeq_next(struct toepcb *toep) { - if ((sk == NULL) || (sk->so_emuldata == NULL)) - return; + struct iscsi_socket *isock = toep->ulpcb; + struct mbuf *m; - switch (op) { - case CPL_ISCSI_HDR: - process_rx_iscsi_hdr(sk, m); - break; - case CPL_RX_DATA_DDP: - process_rx_data_ddp(sk, m); - break; - case CPL_SET_TCB_RPL: - break; - case CPL_FW4_ACK: - process_fw4_ack(sk, m); - break; - default: - CTR2(KTR_CXGBE, "sk 0x%p, op 0x%x from TOM, NOT supported.\n", - sk, op); - break; - } + mtx_lock(&isock->ulp2_writeq_lock); + m = mbufq_dequeue(&isock->ulp2_writeq); + mtx_unlock(&isock->ulp2_writeq_lock); + + mtx_lock(&isock->ulp2_wrq_lock); + mbufq_enqueue(&isock->ulp2_wrq, m); + mtx_unlock(&isock->ulp2_wrq_lock); + + return (mbufq_first(&isock->ulp2_writeq)); } static void t4_register_cpl_handler_with_tom(struct adapter *sc) { - t4tom_register_cpl_iscsi_callback(iscsi_cpl_handler_callback); - t4tom_register_queue_iscsi_callback(iscsi_queue_handler_callback); + t4_register_cpl_handler(sc, CPL_ISCSI_HDR, do_rx_iscsi_hdr); t4_register_cpl_handler(sc, CPL_ISCSI_DATA, do_rx_iscsi_hdr); - t4tom_cpl_handler_register_flag |= - 1 << TOM_CPL_ISCSI_HDR_REGISTERED_BIT; - - if (!t4tom_cpl_handler_registered(sc, CPL_SET_TCB_RPL)) { - t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, do_set_tcb_rpl); - t4tom_cpl_handler_register_flag |= - 1 << TOM_CPL_SET_TCB_RPL_REGISTERED_BIT; - CTR0(KTR_CXGBE, "register t4 cpl handler CPL_SET_TCB_RPL.\n"); - } - t4_register_cpl_handler(sc, CPL_RX_ISCSI_DDP, do_rx_iscsi_ddp); - - if (!t4tom_cpl_handler_registered(sc, CPL_RX_DATA_DDP)) { - t4_register_cpl_handler(sc, CPL_RX_DATA_DDP, do_rx_data_ddp); - t4tom_cpl_handler_register_flag |= - 1 << TOM_CPL_RX_DATA_DDP_REGISTERED_BIT; - CTR0(KTR_CXGBE, "register t4 cpl handler CPL_RX_DATA_DDP.\n"); - } } static void t4_unregister_cpl_handler_with_tom(struct adapter *sc) { - /* de-register CPL handles */ - t4tom_register_cpl_iscsi_callback(NULL); - t4tom_register_queue_iscsi_callback(NULL); - if (t4tom_cpl_handler_register_flag & - (1 << TOM_CPL_ISCSI_HDR_REGISTERED_BIT)) { - t4_register_cpl_handler(sc, CPL_ISCSI_HDR, NULL); - t4_register_cpl_handler(sc, CPL_ISCSI_DATA, NULL); - } - if (t4tom_cpl_handler_register_flag & - (1 << TOM_CPL_SET_TCB_RPL_REGISTERED_BIT)) - t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, NULL); + + t4_register_cpl_handler(sc, CPL_ISCSI_HDR, NULL); + t4_register_cpl_handler(sc, CPL_ISCSI_DATA, NULL); t4_register_cpl_handler(sc, CPL_RX_ISCSI_DDP, NULL); - if (t4tom_cpl_handler_register_flag & - (1 << TOM_CPL_RX_DATA_DDP_REGISTERED_BIT)) - t4_register_cpl_handler(sc, CPL_RX_DATA_DDP, NULL); } static int -send_set_tcb_field(struct socket *sk, u16 word, u64 mask, u64 val, - int no_reply) +send_set_tcb_field(struct toepcb * toep, uint16_t word, uint64_t mask, + uint64_t val, int no_reply) { struct wrqe *wr; struct cpl_set_tcb_field *req; - struct inpcb *inp = sotoinpcb(sk); - struct tcpcb *tp = intotcpcb(inp); - struct toepcb *toep = tp->t_toe; wr = alloc_wrqe(sizeof(*req), toep->ctrlq); if (wr == NULL) @@ -1173,58 +924,23 @@ send_set_tcb_field(struct socket *sk, u16 word, u64 mask, u64 val, req->val = htobe64(val); t4_wrq_tx(toep->td->tod.tod_softc, wr); - return 0; + + return (0); } static int -cxgbei_set_ulp_mode(struct socket *so, struct toepcb *toep, - unsigned char hcrc, unsigned char dcrc) +cxgbei_set_ulp_mode(struct toepcb *toep, u_char hcrc, u_char dcrc) { - int rv = 0, val = 0; + int val = 0; - toep->ulp_mode = ULP_MODE_ISCSI; if (hcrc) val |= ULP_CRC_HEADER; if (dcrc) val |= ULP_CRC_DATA; val <<= 4; val |= ULP_MODE_ISCSI; - rv = send_set_tcb_field(so, 0, 0xfff, val, 0); - return rv; -} -static offload_device * -add_cxgbei_dev(struct ifnet *dev, struct toedev *tdev) -{ -#ifdef T4_DDP - struct cxgbei_ulp2_ddp_info *ddp; -#endif - offload_device *odev = NULL; - odev = offload_device_new(tdev); - if (odev == NULL) { - printf("%s: odev is NULL\n", __func__); - return odev; - } - printf("%s:New T4 %s, tdev 0x%p, odev 0x%p.\n", - __func__, dev->if_xname, tdev, odev); - odev->d_tdev = tdev; - odev->d_ulp_rx_datagap = sizeof(struct cpl_iscsi_hdr_no_rss); - odev->d_flag = ODEV_FLAG_ULP_CRC_ENABLED; - -#ifdef T4_DDP - odev->tdev2ddp = t4_tdev2ddp; - ddp = t4_ddp_init(dev, tdev); - if (ddp) { - printf("T4 %s, odev 0x%p, ddp 0x%p initialized.\n", - dev->if_xname, odev, ddp); - - odev->d_flag |= ODEV_FLAG_ULP_DDP_ENABLED; - cxgbei_ulp2_adapter_ddp_info(ddp, - (struct cxgbei_ulp2_tag_format *)&odev->d_tag_format, - &odev->d_payload_tmax, &odev->d_payload_rmax); - } -#endif - return odev; + return (send_set_tcb_field(toep, 0, 0xfff, val, 1)); } /* initiator */ @@ -1255,26 +971,18 @@ void cxgbei_cleanup_task(void *conn, void *ofld_priv) { struct icl_conn *ic = (struct icl_conn *)conn; - cxgbei_task_data *tdata = NULL; - struct socket *so = NULL; - iscsi_socket *isock = NULL; - offload_device *odev = NULL; + struct cxgbei_task_data *tdata = ofld_priv; + struct iscsi_socket *isock = ic->ic_ofld_prv0; + struct toepcb *toep = isock->toep; + struct adapter *sc = td_adapter(toep->td); + struct cxgbei_data *ci = sc->iscsi_softc; - if (ic->ic_socket == NULL) return; + MPASS(isock != NULL); + MPASS(tdata != NULL); - so = ic->ic_socket; - - isock = (iscsi_socket *)(so)->so_emuldata; - if (isock == NULL) return; - odev = isock->s_odev; - - tdata = (cxgbei_task_data *)(ofld_priv); - if (tdata == NULL) return; - - if (cxgbei_ulp2_is_ddp_tag(&odev->d_tag_format, tdata->sc_ddp_tag)) + if (cxgbei_ulp2_is_ddp_tag(&ci->tag_format, tdata->sc_ddp_tag)) t4_sk_ddp_tag_release(isock, tdata->sc_ddp_tag); memset(tdata, 0, sizeof(*tdata)); - return; } static void @@ -1297,194 +1005,266 @@ t4_sk_tx_mbuf_setmode(struct icl_pdu *req, void *toep, void *mbuf, } int -cxgbei_conn_xmit_pdu(void *conn, void *ioreq) +cxgbei_conn_xmit_pdu(struct icl_conn *ic, struct icl_pdu *req) { - struct icl_conn *ic = (struct icl_conn *)conn; - struct icl_pdu *req = (struct icl_pdu *)ioreq; struct mbuf *m = req->ip_bhs_mbuf; - struct socket *so = ic->ic_socket; - struct tcpcb *tp = so_sototcpcb(so); + struct iscsi_socket *isock = ic->ic_ofld_prv0; + struct toepcb *toep = isock->toep; - t4_sk_tx_mbuf_setmode(req, tp->t_toe, m, 2, - ic->ic_header_crc32c ? ISCSI_HEADER_DIGEST_SIZE : 0, - (req->ip_data_len && ic->ic_data_crc32c) ? ISCSI_DATA_DIGEST_SIZE : 0); + t4_sk_tx_mbuf_setmode(req, toep, m, 2, + ic->ic_header_crc32c ? ISCSI_HEADER_DIGEST_SIZE : 0, + (req->ip_data_len && ic->ic_data_crc32c) ? ISCSI_DATA_DIGEST_SIZE : 0); - t4_ulp_mbuf_push(ic->ic_socket, m); - return 0; -} - -/* called from host iscsi, socket is passed as argument */ -int -cxgbei_conn_set_ulp_mode(struct socket *so, void *conn) -{ - struct tcpcb *tp = so_sototcpcb(so); - struct toepcb *toep = tp->t_toe; - struct adapter *sc = NULL; - struct toedev *tdev = NULL; - iscsi_socket *isock = NULL; - struct ifnet *ifp = NULL; - unsigned int tid = toep->tid; - offload_device *odev = NULL; - struct icl_conn *ic = (struct icl_conn*)conn; - - if (toep == NULL) return EINVAL; - - ifp = toep->port->ifp; - if (ifp == NULL) return EINVAL; - - if (!(sototcpcb(so)->t_flags & TF_TOE) || - !(ifp->if_capenable & IFCAP_TOE)) { - printf("%s: TOE not enabled on:%s\n", __func__, ifp->if_xname); - return EINVAL; - } - - /* if ULP_MODE is set by TOE driver, treat it as non-offloaded */ - if (toep->ulp_mode) { - CTR3(KTR_CXGBE, "%s: T4 sk 0x%p, ulp mode already set 0x%x.\n", - __func__, so, toep->ulp_mode); - return EINVAL; - } - sc = toep->port->adapter; - tdev = &toep->td->tod; - /* if toe dev is not set, treat it as non-offloaded */ - if (tdev == NULL) { - CTR2(KTR_CXGBE, "%s: T4 sk 0x%p, tdev NULL.\n", __func__, so); - return EINVAL; - } - - isock = (iscsi_socket *)malloc(sizeof(iscsi_socket), M_CXGBE, - M_NOWAIT | M_ZERO); - if (isock == NULL) { - printf("%s: T4 sk 0x%p, isock alloc failed.\n", __func__, so); - return EINVAL; - } - isock->mbuf_ulp_lhdr = NULL; - isock->sock = so; - isock->s_conn = conn; - so->so_emuldata = isock; - - mtx_init(&isock->iscsi_rcv_mbufq.lock,"isock_lock" , NULL, MTX_DEF); - mtx_init(&isock->ulp2_wrq.lock,"ulp2_wrq lock" , NULL, MTX_DEF); - mtx_init(&isock->ulp2_writeq.lock,"ulp2_writeq lock" , NULL, MTX_DEF); - - CTR6(KTR_CXGBE, - "%s: sc:%p toep:%p iscsi_start:0x%x iscsi_size:0x%x caps:%d.\n", - __func__, sc, toep, sc->vres.iscsi.start, - sc->vres.iscsi.size, sc->iscsicaps); - /* - * Register ULP CPL handlers with TOM - * Register CPL_RX_ISCSI_HDR, CPL_RX_DATA_DDP callbacks with TOM - */ - t4_register_cpl_handler_with_tom(sc); - - /* - * DDP initialization. Once for each tdev - * check if DDP is already configured for this tdev - */ - odev = offload_device_find(tdev); - if (odev == NULL) /* for each tdev we have a corresponding odev */ - { - if ((odev = add_cxgbei_dev(ifp, tdev)) == NULL) { - CTR3(KTR_CXGBE, - "T4 sk 0x%p, tdev %s, 0x%p, odev NULL.\n", - so, ifp->if_xname, tdev); - return EINVAL; - } - } - - CTR3(KTR_CXGBE, "tdev:%p sc->iscsi_softc:%p odev:%p\n", - tdev, sc->iscsi_softc, odev); - isock->s_odev = odev; - isock->s_tid = tid; - - isock->s_rmax = odev->d_payload_rmax; - isock->s_tmax = odev->d_payload_tmax; - - /* XXX cap the xmit pdu size to be 12K for now until f/w is ready */ - if (isock->s_tmax > (12288 + ISCSI_PDU_NONPAYLOAD_LEN)) - isock->s_tmax = 12288 + ISCSI_PDU_NONPAYLOAD_LEN; - - /* set toe DDP off */ - so->so_options |= SO_NO_DDP; - - /* Move connection to ULP mode, SET_TCB_FIELD */ - cxgbei_set_ulp_mode(so, toep, - ic->ic_header_crc32c, ic->ic_data_crc32c); - - isock->s_hcrc_len = (ic->ic_header_crc32c ? 4 : 0); - isock->s_dcrc_len = (ic->ic_data_crc32c ? 4 : 0); - return 0; + t4_ulp_mbuf_push(isock, m); + return (0); } int -cxgbei_conn_close(struct socket *so) +cxgbei_conn_handoff(struct icl_conn *ic) { - iscsi_socket *isock = NULL; - isock = (iscsi_socket *)(so)->so_emuldata; + struct tcpcb *tp = so_sototcpcb(ic->ic_socket); + struct toepcb *toep; + struct iscsi_socket *isock; + + if (!(tp->t_flags & TF_TOE)) + return (ENOTSUP); /* Connection is not offloaded. */ + MPASS(tp->tod != NULL); + MPASS(tp->t_toe != NULL); + + /* + * XXXNP: Seems broken. How can we assume that the tod/toep is what we + * think it is? + */ + + toep = tp->t_toe; + if (toep->ulp_mode) + return (EBUSY); /* Stay away if ulp_mode is already set. */ + + isock = malloc(sizeof(struct iscsi_socket), M_CXGBE, M_NOWAIT | M_ZERO); + if (isock == NULL) + return (ENOMEM); + isock->s_conn = ic; + isock->toep = toep; + isock->s_dcrc_len = ic->ic_data_crc32c ? 4 : 0; + + mbufq_init(&isock->iscsi_rcvq, INT_MAX); + mtx_init(&isock->iscsi_rcvq_lock,"isock_lock" , NULL, MTX_DEF); + + mbufq_init(&isock->ulp2_wrq, INT_MAX); + mtx_init(&isock->ulp2_wrq_lock,"ulp2_wrq lock" , NULL, MTX_DEF); + + mbufq_init(&isock->ulp2_writeq, INT_MAX); + mtx_init(&isock->ulp2_writeq_lock,"ulp2_writeq lock" , NULL, MTX_DEF); + + /* Move connection to ULP mode. */ + ic->ic_socket->so_options |= SO_NO_DDP; + toep->ulp_mode = ULP_MODE_ISCSI; + toep->ulpcb = isock; + ic->ic_ofld_prv0 = isock; + + return (cxgbei_set_ulp_mode(toep, ic->ic_header_crc32c, ic->ic_data_crc32c)); +} + +int +cxgbei_conn_close(struct icl_conn *ic) +{ + struct iscsi_socket *isock = ic->ic_ofld_prv0; + struct toepcb *toep = isock->toep; struct mbuf *m; struct ulp_mbuf_cb *cb; struct icl_pdu *req; - so->so_emuldata = NULL; + MPASS(isock != NULL); /* free isock Qs */ - while ((m = mbufq_dequeue(&isock->iscsi_rcv_mbufq)) != NULL) - m_freem(m); + /* + * XXXNP: some drained with lock held, some without. And the test for + * whether the lock has even been initialized is after it has been + * grabbed and released already. + * + * An even larger issue is whether the TCP connection is going down + * gracefully or not. Can't simply throw away stuff in send/rcv buffers + * if the TCP shutdown is supposed to be graceful. + */ + mbufq_drain(&isock->iscsi_rcvq); + mbufq_drain(&isock->ulp2_writeq); - while ((m = mbufq_dequeue(&isock->ulp2_writeq)) != NULL) - m_freem(m); - - mtx_lock(&isock->ulp2_wrq.lock); + mtx_lock(&isock->ulp2_wrq_lock); while ((m = mbufq_dequeue(&isock->ulp2_wrq)) != NULL) { cb = find_ulp_mbuf_cb(m); - if (cb && isock && cb->pdu) { + if (cb && cb->pdu) { req = (struct icl_pdu *)cb->pdu; req->ip_bhs_mbuf = NULL; icl_pdu_free(req); } m_freem(m); } - mtx_unlock(&isock->ulp2_wrq.lock); + mtx_unlock(&isock->ulp2_wrq_lock); - if (mtx_initialized(&isock->iscsi_rcv_mbufq.lock)) - mtx_destroy(&isock->iscsi_rcv_mbufq.lock); + if (mtx_initialized(&isock->iscsi_rcvq_lock)) + mtx_destroy(&isock->iscsi_rcvq_lock); - if (mtx_initialized(&isock->ulp2_wrq.lock)) - mtx_destroy(&isock->ulp2_wrq.lock); + if (mtx_initialized(&isock->ulp2_wrq_lock)) + mtx_destroy(&isock->ulp2_wrq_lock); - if (mtx_initialized(&isock->ulp2_writeq.lock)) - mtx_destroy(&isock->ulp2_writeq.lock); + if (mtx_initialized(&isock->ulp2_writeq_lock)) + mtx_destroy(&isock->ulp2_writeq_lock); + + /* XXXNP: Should the ulpcb and ulp_mode be cleared here? */ + toep->ulp_mode = ULP_MODE_NONE; /* dubious without inp lock */ free(isock, M_CXGBE); - return 0; + return (0); } static int -cxgbei_loader(struct module *mod, int cmd, void *arg) +cxgbei_activate(struct adapter *sc) { - int err = 0; + struct cxgbei_data *ci; + int rc; + + ASSERT_SYNCHRONIZED_OP(sc); + + if (uld_active(sc, ULD_ISCSI)) { + KASSERT(0, ("%s: iSCSI offload already enabled on adapter %p", + __func__, sc)); + return (0); + } + + if (sc->iscsicaps == 0 || sc->vres.iscsi.size == 0) { + device_printf(sc->dev, + "not iSCSI offload capable, or capability disabled.\n"); + return (ENOSYS); + } + + /* per-adapter softc for iSCSI */ + ci = malloc(sizeof(*ci), M_CXGBE, M_ZERO | M_NOWAIT); + if (ci == NULL) + return (ENOMEM); + + rc = cxgbei_ddp_init(sc, ci); + if (rc != 0) { + free(ci, M_CXGBE); + return (rc); + } + + t4_register_cpl_handler_with_tom(sc); + sc->iscsi_softc = ci; + + return (0); +} + +static int +cxgbei_deactivate(struct adapter *sc) +{ + + ASSERT_SYNCHRONIZED_OP(sc); + + if (sc->iscsi_softc != NULL) { + cxgbei_ddp_cleanup(sc->iscsi_softc); + t4_unregister_cpl_handler_with_tom(sc); + free(sc->iscsi_softc, M_CXGBE); + sc->iscsi_softc = NULL; + } + + return (0); +} + +static void +cxgbei_activate_all(struct adapter *sc, void *arg __unused) +{ + + if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4isact") != 0) + return; + + /* Activate iSCSI if any port on this adapter has IFCAP_TOE enabled. */ + if (sc->offload_map && !uld_active(sc, ULD_ISCSI)) + (void) t4_activate_uld(sc, ULD_ISCSI); + + end_synchronized_op(sc, 0); +} + +static void +cxgbei_deactivate_all(struct adapter *sc, void *arg __unused) +{ + + if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4isdea") != 0) + return; + + if (uld_active(sc, ULD_ISCSI)) + (void) t4_deactivate_uld(sc, ULD_ISCSI); + + end_synchronized_op(sc, 0); +} + +static struct uld_info cxgbei_uld_info = { + .uld_id = ULD_ISCSI, + .activate = cxgbei_activate, + .deactivate = cxgbei_deactivate, +}; + +extern void (*cxgbei_fw4_ack)(struct toepcb *, int); +extern void (*cxgbei_rx_data_ddp)(struct toepcb *, + const struct cpl_rx_data_ddp *); +extern struct mbuf *(*cxgbei_writeq_len)(struct toepcb *, int *); +extern struct mbuf *(*cxgbei_writeq_next)(struct toepcb *); + +static int +cxgbei_mod_load(void) +{ + int rc; + + cxgbei_fw4_ack = drop_fw_acked_ulp_data; + cxgbei_rx_data_ddp = process_rx_data_ddp; + cxgbei_writeq_len = get_writeq_len; + cxgbei_writeq_next = do_writeq_next; + + rc = t4_register_uld(&cxgbei_uld_info); + if (rc != 0) + return (rc); + + t4_iterate(cxgbei_activate_all, NULL); + + return (rc); +} + +static int +cxgbei_mod_unload(void) +{ + + t4_iterate(cxgbei_deactivate_all, NULL); + + if (t4_unregister_uld(&cxgbei_uld_info) == EBUSY) + return (EBUSY); + + return (0); +} + +static int +cxgbei_modevent(module_t mod, int cmd, void *arg) +{ + int rc = 0; switch (cmd) { case MOD_LOAD: - SLIST_INIT(&odev_list); - printf("cxgbei module loaded Sucessfully.\n"); + rc = cxgbei_mod_load(); break; + case MOD_UNLOAD: - offload_device_remove(); - printf("cxgbei cleanup completed sucessfully.\n"); + rc = cxgbei_mod_unload(); break; + default: - err = (EINVAL); - break; + rc = EINVAL; } - return (err); + return (rc); } static moduledata_t cxgbei_mod = { "cxgbei", - cxgbei_loader, + cxgbei_modevent, NULL, }; diff --git a/sys/dev/cxgbe/cxgbei/cxgbei.h b/sys/dev/cxgbe/cxgbei/cxgbei.h index 0415ecaf399..4df28bc35c2 100644 --- a/sys/dev/cxgbe/cxgbei/cxgbei.h +++ b/sys/dev/cxgbe/cxgbei/cxgbei.h @@ -28,55 +28,37 @@ #ifndef __CXGBEI_OFLD_H__ #define __CXGBEI_OFLD_H__ -#include "mbufq.h" -typedef struct iscsi_socket { - /* iscsi private */ - unsigned char s_flag; - unsigned char s_cpuno; /* bind to cpuno */ - unsigned char s_mode; /* offload mode */ - unsigned char s_txhold; +struct iscsi_socket { + u_char s_dcrc_len; + void *s_conn; /* ic_conn pointer */ + struct toepcb *toep; - unsigned char s_ddp_pgidx; /* ddp page selection */ - unsigned char s_hcrc_len; - unsigned char s_dcrc_len; - unsigned char filler[1]; + /* + * XXXNP: locks on the same line. + * XXXNP: are the locks even needed? Why not use so_snd/so_rcv mtx to + * guard the write and rcv queues? + */ + struct mbufq iscsi_rcvq; /* rx - ULP mbufs */ + struct mtx iscsi_rcvq_lock; - unsigned int s_tid; /* for debug only */ - unsigned int s_tmax; - unsigned int s_rmax; - unsigned int s_mss; - void *s_odev; /* offload device, if any */ - void *s_appdata; /* upperlayer data pointer */ - void *s_private; /* underlying socket related info. */ - void *s_conn; /* ic_conn pointer */ - struct socket *sock; - struct mbuf_head iscsi_rcv_mbufq;/* rx - ULP mbufs */ - struct mbuf_head ulp2_writeq; /* tx - ULP mbufs */ - struct mbuf_head ulp2_wrq; /* tx wr- ULP mbufs */ + struct mbufq ulp2_writeq; /* tx - ULP mbufs */ + struct mtx ulp2_writeq_lock; + + struct mbufq ulp2_wrq; /* tx wr- ULP mbufs */ + struct mtx ulp2_wrq_lock; struct mbuf *mbuf_ulp_lhdr; struct mbuf *mbuf_ulp_ldata; -}iscsi_socket; +}; -#define ISCSI_SG_SBUF_DMABLE 0x1 -#define ISCSI_SG_SBUF_DMA_ONLY 0x2 /*private*/ -#define ISCSI_SG_BUF_ALLOC 0x10 -#define ISCSI_SG_PAGE_ALLOC 0x20 -#define ISCSI_SG_SBUF_MAP_NEEDED 0x40 -#define ISCSI_SG_SBUF_MAPPED 0x80 - -#define ISCSI_SG_SBUF_LISTHEAD 0x100 -#define ISCSI_SG_SBUF_LISTTAIL 0x200 -#define ISCSI_SG_SBUF_XFER_DONE 0x400 - -typedef struct cxgbei_sgl { +struct cxgbei_sgl { int sg_flag; void *sg_addr; void *sg_dma_addr; size_t sg_offset; size_t sg_length; -} cxgbei_sgl; +}; #define cxgbei_scsi_for_each_sg(_sgl, _sgel, _n, _i) \ for (_i = 0, _sgel = (cxgbei_sgl*) (_sgl); _i < _n; _i++, \ @@ -96,14 +78,6 @@ typedef struct cxgbei_sgl { #define SBUF_ULP_FLAG_PAD_ERROR 0x40 #define SBUF_ULP_FLAG_DATA_DDPED 0x80 -/* Flags for return value of CPL message handlers */ -enum { - CPL_RET_BUF_DONE = 1, /* buffer processing done buffer may be freed */ - CPL_RET_BAD_MSG = 2, /* bad CPL message (e.g., unknown opcode) */ - CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */ -}; - - /* * Similar to tcp_skb_cb but with ULP elements added to support DDP, iSCSI, * etc. @@ -126,33 +100,55 @@ struct ulp_mbuf_cb { void *pdu; /* pdu pointer */ }; -/* private data for eack scsi task */ -typedef struct cxgbei_task_data { - cxgbei_sgl sgl[256]; - unsigned int nsge; - unsigned int sc_ddp_tag; -}cxgbei_task_data; - -static unsigned char t4tom_cpl_handler_register_flag; -enum { - TOM_CPL_ISCSI_HDR_REGISTERED_BIT, - TOM_CPL_SET_TCB_RPL_REGISTERED_BIT, - TOM_CPL_RX_DATA_DDP_REGISTERED_BIT +/* private data for each scsi task */ +struct cxgbei_task_data { + struct cxgbei_sgl sgl[256]; + u_int nsge; + u_int sc_ddp_tag; }; -#define ODEV_FLAG_ULP_CRC_ENABLED 0x1 -#define ODEV_FLAG_ULP_DDP_ENABLED 0x2 -#define ODEV_FLAG_ULP_TX_ALLOC_DIGEST 0x4 -#define ODEV_FLAG_ULP_RX_PAD_INCLUDED 0x8 +struct cxgbei_ulp2_tag_format { + u_char sw_bits; + u_char rsvd_bits; + u_char rsvd_shift; + u_char filler[1]; + uint32_t rsvd_mask; +}; -#define ODEV_FLAG_ULP_ENABLED \ - (ODEV_FLAG_ULP_CRC_ENABLED | ODEV_FLAG_ULP_DDP_ENABLED) +struct cxgbei_data { + u_int max_txsz; + u_int max_rxsz; + u_int llimit; + u_int ulimit; + u_int nppods; + u_int idx_last; + u_char idx_bits; + uint32_t idx_mask; + uint32_t rsvd_tag_mask; -struct ulp_mbuf_cb * get_ulp_mbuf_cb(struct mbuf *); -int cxgbei_conn_set_ulp_mode(struct socket *, void *); -int cxgbei_conn_close(struct socket *); + struct mtx map_lock; + bus_dma_tag_t ulp_ddp_tag; + unsigned char *colors; + struct cxgbei_ulp2_gather_list **gl_map; + + struct cxgbei_ulp2_tag_format tag_format; +}; + +struct icl_conn; +struct icl_pdu; + +struct ulp_mbuf_cb *get_ulp_mbuf_cb(struct mbuf *); +int cxgbei_conn_handoff(struct icl_conn *); +int cxgbei_conn_close(struct icl_conn *); void cxgbei_conn_task_reserve_itt(void *, void **, void *, unsigned int *); void cxgbei_conn_transfer_reserve_ttt(void *, void **, void *, unsigned int *); void cxgbei_cleanup_task(void *, void *); -int cxgbei_conn_xmit_pdu(void *, void *); +int cxgbei_conn_xmit_pdu(struct icl_conn *, struct icl_pdu *); + +struct cxgbei_ulp2_pagepod_hdr; +int t4_ddp_set_map(struct cxgbei_data *, void *, + struct cxgbei_ulp2_pagepod_hdr *, u_int, u_int, + struct cxgbei_ulp2_gather_list *, int); +void t4_ddp_clear_map(struct cxgbei_data *, struct cxgbei_ulp2_gather_list *, + u_int, u_int, u_int, struct iscsi_socket *); #endif diff --git a/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.c b/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.c index 94c94ffc676..f99ce012478 100644 --- a/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.c +++ b/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.c @@ -60,27 +60,6 @@ __FBSDID("$FreeBSD$"); #include "cxgbei.h" #include "cxgbei_ulp2_ddp.h" -static inline int -cxgbei_counter_dec_and_read(volatile int *p) -{ - atomic_subtract_acq_int(p, 1); - return atomic_load_acq_int(p); -} - -static inline int -get_order(unsigned long size) -{ - int order; - - size = (size - 1) >> PAGE_SHIFT; - order = 0; - while (size) { - order++; - size >>= 1; - } - return (order); -} - /* * Map a single buffer address. */ @@ -96,25 +75,6 @@ ulp2_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error) *ba = segs->ds_addr; } -static int -ulp2_dma_tag_create(struct cxgbei_ulp2_ddp_info *ddp) -{ - int rc; - - rc = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR, - BUS_SPACE_MAXADDR, NULL, NULL, UINT32_MAX , 8, - BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL, NULL, - &ddp->ulp_ddp_tag); - - if (rc != 0) { - printf("%s(%d): bus_dma_tag_create() " - "failed (rc = %d)!\n", - __FILE__, __LINE__, rc); - return rc; - } - return 0; -} - /* * iSCSI Direct Data Placement * @@ -129,157 +89,95 @@ ulp2_dma_tag_create(struct cxgbei_ulp2_ddp_info *ddp) * is the base for ITT/TTT. */ -unsigned char ddp_page_order[DDP_PGIDX_MAX] = {0, 1, 2, 4}; -unsigned char ddp_page_shift[DDP_PGIDX_MAX] = {12, 13, 14, 16}; -unsigned char page_idx = DDP_PGIDX_MAX; static inline int -ddp_find_unused_entries(struct cxgbei_ulp2_ddp_info *ddp, - unsigned int start, unsigned int max, - unsigned int count, unsigned int *idx, - struct cxgbei_ulp2_gather_list *gl) +ddp_find_unused_entries(struct cxgbei_data *ci, u_int start, u_int max, + u_int count, u_int *idx, struct cxgbei_ulp2_gather_list *gl) { unsigned int i, j, k; /* not enough entries */ - if ((max - start) < count) - return EBUSY; + if (max - start < count) + return (EBUSY); max -= count; - mtx_lock(&ddp->map_lock); + mtx_lock(&ci->map_lock); for (i = start; i < max;) { for (j = 0, k = i; j < count; j++, k++) { - if (ddp->gl_map[k]) + if (ci->gl_map[k]) break; } if (j == count) { for (j = 0, k = i; j < count; j++, k++) - ddp->gl_map[k] = gl; - mtx_unlock(&ddp->map_lock); + ci->gl_map[k] = gl; + mtx_unlock(&ci->map_lock); *idx = i; - return 0; + return (0); } i += j + 1; } - mtx_unlock(&ddp->map_lock); - return EBUSY; + mtx_unlock(&ci->map_lock); + return (EBUSY); } static inline void -ddp_unmark_entries(struct cxgbei_ulp2_ddp_info *ddp, - int start, int count) +ddp_unmark_entries(struct cxgbei_data *ci, u_int start, u_int count) { - mtx_lock(&ddp->map_lock); - memset(&ddp->gl_map[start], 0, + + mtx_lock(&ci->map_lock); + memset(&ci->gl_map[start], 0, count * sizeof(struct cxgbei_ulp2_gather_list *)); - mtx_unlock(&ddp->map_lock); + mtx_unlock(&ci->map_lock); } -/** - * cxgbei_ulp2_ddp_find_page_index - return ddp page index for a given page size - * @pgsz: page size - * return the ddp page index, if no match is found return DDP_PGIDX_MAX. - */ -int -cxgbei_ulp2_ddp_find_page_index(unsigned long pgsz) -{ - int i; - - for (i = 0; i < DDP_PGIDX_MAX; i++) { - if (pgsz == (1UL << ddp_page_shift[i])) - return i; - } - CTR1(KTR_CXGBE, "ddp page size 0x%lx not supported.\n", pgsz); - return DDP_PGIDX_MAX; -} - -static int -cxgbei_ulp2_ddp_adjust_page_table(void) -{ - int i; - unsigned int base_order, order; - - if (PAGE_SIZE < (1UL << ddp_page_shift[0])) { - CTR2(KTR_CXGBE, "PAGE_SIZE %u too small, min. %lu.\n", - PAGE_SIZE, 1UL << ddp_page_shift[0]); - return EINVAL; - } - - base_order = get_order(1UL << ddp_page_shift[0]); - order = get_order(1 << PAGE_SHIFT); - for (i = 0; i < DDP_PGIDX_MAX; i++) { - /* first is the kernel page size, then just doubling the size */ - ddp_page_order[i] = order - base_order + i; - ddp_page_shift[i] = PAGE_SHIFT + i; - } - return 0; -} - - static inline void -ddp_gl_unmap(struct toedev *tdev, - struct cxgbei_ulp2_gather_list *gl) +ddp_gl_unmap(struct cxgbei_data *ci, struct cxgbei_ulp2_gather_list *gl) { int i; - struct adapter *sc = tdev->tod_softc; - struct cxgbei_ulp2_ddp_info *ddp = sc->iscsi_softc; if (!gl->pages[0]) return; for (i = 0; i < gl->nelem; i++) { - bus_dmamap_unload(ddp->ulp_ddp_tag, gl->dma_sg[i].bus_map); - bus_dmamap_destroy(ddp->ulp_ddp_tag, gl->dma_sg[i].bus_map); + bus_dmamap_unload(ci->ulp_ddp_tag, gl->dma_sg[i].bus_map); + bus_dmamap_destroy(ci->ulp_ddp_tag, gl->dma_sg[i].bus_map); } } static inline int -ddp_gl_map(struct toedev *tdev, - struct cxgbei_ulp2_gather_list *gl) +ddp_gl_map(struct cxgbei_data *ci, struct cxgbei_ulp2_gather_list *gl) { int i, rc; bus_addr_t pa; - struct cxgbei_ulp2_ddp_info *ddp; - struct adapter *sc = tdev->tod_softc; - ddp = (struct cxgbei_ulp2_ddp_info *)sc->iscsi_softc; - if (ddp == NULL) { - printf("%s: DDP is NULL tdev:%p sc:%p ddp:%p\n", - __func__, tdev, sc, ddp); - return ENOMEM; - } - mtx_lock(&ddp->map_lock); + MPASS(ci != NULL); + + mtx_lock(&ci->map_lock); for (i = 0; i < gl->nelem; i++) { - rc = bus_dmamap_create(ddp->ulp_ddp_tag, 0, - &gl->dma_sg[i].bus_map); - if (rc != 0) { - printf("%s: unable to map page 0x%p.\n", - __func__, gl->pages[i]); + rc = bus_dmamap_create(ci->ulp_ddp_tag, 0, + &gl->dma_sg[i].bus_map); + if (rc != 0) goto unmap; - } - rc = bus_dmamap_load(ddp->ulp_ddp_tag, gl->dma_sg[i].bus_map, + rc = bus_dmamap_load(ci->ulp_ddp_tag, gl->dma_sg[i].bus_map, gl->pages[i], PAGE_SIZE, ulp2_dma_map_addr, &pa, BUS_DMA_NOWAIT); - if (rc != 0) { - printf("%s:unable to load page 0x%p.\n", - __func__, gl->pages[i]); + if (rc != 0) goto unmap; - } gl->dma_sg[i].phys_addr = pa; } - mtx_unlock(&ddp->map_lock); + mtx_unlock(&ci->map_lock); - return 0; + return (0); unmap: if (i) { - unsigned int nelem = gl->nelem; + u_int nelem = gl->nelem; gl->nelem = i; - ddp_gl_unmap(tdev, gl); + ddp_gl_unmap(ci, gl); gl->nelem = nelem; } - return ENOMEM; + return (ENOMEM); } /** @@ -297,13 +195,11 @@ unmap: * memory can be used for ddp. Return NULL otherwise. */ struct cxgbei_ulp2_gather_list * -cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec - (unsigned int xferlen, cxgbei_sgl *sgl, - unsigned int sgcnt, void *tdev, - int gfp) +cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(u_int xferlen, struct cxgbei_sgl *sgl, + u_int sgcnt, struct cxgbei_data *ci, int gfp) { struct cxgbei_ulp2_gather_list *gl; - cxgbei_sgl *sg = sgl; + struct cxgbei_sgl *sg = sgl; void *sgpage = (void *)((u64)sg->sg_addr & (~PAGE_MASK)); unsigned int sglen = sg->sg_length; unsigned int sgoffset = (u64)sg->sg_addr & PAGE_MASK; @@ -320,10 +216,8 @@ cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec gl = malloc(sizeof(struct cxgbei_ulp2_gather_list) + npages * (sizeof(struct dma_segments) + sizeof(void *)), M_DEVBUF, M_NOWAIT | M_ZERO); - if (gl == NULL) { - printf("%s: gl alloc failed\n", __func__); - return NULL; - } + if (gl == NULL) + return (NULL); gl->pages = (void **)&gl->dma_sg[npages]; gl->length = xferlen; @@ -361,7 +255,7 @@ cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec } gl->nelem = ++j; - if (ddp_gl_map(tdev, gl) < 0) + if (ddp_gl_map(ci, gl) < 0) goto error_out; return gl; @@ -378,15 +272,17 @@ error_out: * free a ddp page buffer list resulted from cxgbei_ulp2_ddp_make_gl(). */ void -cxgbei_ulp2_ddp_release_gl(struct cxgbei_ulp2_gather_list *gl, void *tdev) +cxgbei_ulp2_ddp_release_gl(struct cxgbei_data *ci, + struct cxgbei_ulp2_gather_list *gl) { - ddp_gl_unmap(tdev, gl); + + ddp_gl_unmap(ci, gl); free(gl, M_DEVBUF); } /** * cxgbei_ulp2_ddp_tag_reserve - set up ddp for a data transfer - * @ddp: adapter's ddp info + * @ci: adapter's ddp info * @tid: connection id * @tformat: tag format * @tagp: contains s/w tag initially, will be updated with ddp/hw tag @@ -397,93 +293,84 @@ cxgbei_ulp2_ddp_release_gl(struct cxgbei_ulp2_gather_list *gl, void *tdev) * return 0 if success, < 0 otherwise. */ int -cxgbei_ulp2_ddp_tag_reserve(struct cxgbei_ulp2_ddp_info *ddp, - void *isock, unsigned int tid, - struct cxgbei_ulp2_tag_format *tformat, - u32 *tagp, struct cxgbei_ulp2_gather_list *gl, - int gfp, int reply) +cxgbei_ulp2_ddp_tag_reserve(struct cxgbei_data *ci, void *isock, u_int tid, + struct cxgbei_ulp2_tag_format *tformat, u32 *tagp, + struct cxgbei_ulp2_gather_list *gl, int gfp, int reply) { struct cxgbei_ulp2_pagepod_hdr hdr; - unsigned int npods, idx; - int rv; + u_int npods, idx; + int rc; u32 sw_tag = *tagp; u32 tag; - if (page_idx >= DDP_PGIDX_MAX || !ddp || !gl || !gl->nelem || - gl->length < DDP_THRESHOLD) { - CTR3(KTR_CXGBE, "pgidx %u, xfer %u/%u, NO ddp.\n", - page_idx, gl->length, DDP_THRESHOLD); - return EINVAL; - } + MPASS(ci != NULL); + + if (!gl || !gl->nelem || gl->length < DDP_THRESHOLD) + return (EINVAL); npods = (gl->nelem + IPPOD_PAGES_MAX - 1) >> IPPOD_PAGES_SHIFT; - if (ddp->idx_last == ddp->nppods) - rv = ddp_find_unused_entries(ddp, 0, ddp->nppods, - npods, &idx, gl); + if (ci->idx_last == ci->nppods) + rc = ddp_find_unused_entries(ci, 0, ci->nppods, npods, &idx, + gl); else { - rv = ddp_find_unused_entries(ddp, ddp->idx_last + 1, - ddp->nppods, npods, &idx, gl); - if (rv && ddp->idx_last >= npods) { - rv = ddp_find_unused_entries(ddp, 0, - min(ddp->idx_last + npods, ddp->nppods), + rc = ddp_find_unused_entries(ci, ci->idx_last + 1, + ci->nppods, npods, &idx, gl); + if (rc && ci->idx_last >= npods) { + rc = ddp_find_unused_entries(ci, 0, + min(ci->idx_last + npods, ci->nppods), npods, &idx, gl); } } - if (rv) { + if (rc) { CTR3(KTR_CXGBE, "xferlen %u, gl %u, npods %u NO DDP.\n", gl->length, gl->nelem, npods); - return rv; + return (rc); } - tag = cxgbei_ulp2_ddp_tag_base(idx, ddp, tformat, sw_tag); + tag = cxgbei_ulp2_ddp_tag_base(idx, ci->colors, tformat, sw_tag); CTR4(KTR_CXGBE, "%s: sw_tag:0x%x idx:0x%x tag:0x%x\n", __func__, sw_tag, idx, tag); hdr.rsvd = 0; hdr.vld_tid = htonl(F_IPPOD_VALID | V_IPPOD_TID(tid)); - hdr.pgsz_tag_clr = htonl(tag & ddp->rsvd_tag_mask); + hdr.pgsz_tag_clr = htonl(tag & ci->rsvd_tag_mask); hdr.maxoffset = htonl(gl->length); hdr.pgoffset = htonl(gl->offset); - rv = ddp->ddp_set_map(ddp, isock, &hdr, idx, npods, gl, reply); - if (rv < 0) + rc = t4_ddp_set_map(ci, isock, &hdr, idx, npods, gl, reply); + if (rc < 0) goto unmark_entries; - ddp->idx_last = idx; + ci->idx_last = idx; *tagp = tag; - return 0; + return (0); unmark_entries: - ddp_unmark_entries(ddp, idx, npods); - return rv; + ddp_unmark_entries(ci, idx, npods); + return (rc); } /** * cxgbei_ulp2_ddp_tag_release - release a ddp tag - * @ddp: adapter's ddp info + * @ci: adapter's ddp info * @tag: ddp tag * ddp cleanup for a given ddp tag and release all the resources held */ void -cxgbei_ulp2_ddp_tag_release(struct cxgbei_ulp2_ddp_info *ddp, u32 tag, - iscsi_socket *isock) +cxgbei_ulp2_ddp_tag_release(struct cxgbei_data *ci, uint32_t tag, + struct iscsi_socket *isock) { - u32 idx; + uint32_t idx; - if (ddp == NULL) { - CTR2(KTR_CXGBE, "%s:release ddp tag 0x%x, ddp NULL.\n", - __func__, tag); - return; - } - if (isock == NULL) - return; + MPASS(ci != NULL); + MPASS(isock != NULL); - idx = (tag >> IPPOD_IDX_SHIFT) & ddp->idx_mask; + idx = (tag >> IPPOD_IDX_SHIFT) & ci->idx_mask; CTR3(KTR_CXGBE, "tag:0x%x idx:0x%x nppods:0x%x\n", - tag, idx, ddp->nppods); - if (idx < ddp->nppods) { - struct cxgbei_ulp2_gather_list *gl = ddp->gl_map[idx]; + tag, idx, ci->nppods); + if (idx < ci->nppods) { + struct cxgbei_ulp2_gather_list *gl = ci->gl_map[idx]; unsigned int npods; if (!gl || !gl->nelem) { @@ -495,209 +382,32 @@ cxgbei_ulp2_ddp_tag_release(struct cxgbei_ulp2_ddp_info *ddp, u32 tag, npods = (gl->nelem + IPPOD_PAGES_MAX - 1) >> IPPOD_PAGES_SHIFT; CTR3(KTR_CXGBE, "ddp tag 0x%x, release idx 0x%x, npods %u.\n", tag, idx, npods); - ddp->ddp_clear_map(ddp, gl, tag, idx, npods, isock); - ddp_unmark_entries(ddp, idx, npods); - cxgbei_ulp2_ddp_release_gl(gl, ddp->tdev); + t4_ddp_clear_map(ci, gl, tag, idx, npods, isock); + ddp_unmark_entries(ci, idx, npods); + cxgbei_ulp2_ddp_release_gl(ci, gl); } else CTR3(KTR_CXGBE, "ddp tag 0x%x, idx 0x%x > max 0x%x.\n", - tag, idx, ddp->nppods); + tag, idx, ci->nppods); } /** - * cxgbei_ulp2_adapter_ddp_info - read the adapter's ddp information - * @ddp: adapter's ddp info - * @tformat: tag format - * @txsz: max tx pdu payload size, filled in by this func. - * @rxsz: max rx pdu payload size, filled in by this func. - * setup the tag format for a given iscsi entity - */ -int -cxgbei_ulp2_adapter_ddp_info(struct cxgbei_ulp2_ddp_info *ddp, - struct cxgbei_ulp2_tag_format *tformat, - unsigned int *txsz, unsigned int *rxsz) -{ - unsigned char idx_bits; - - if (tformat == NULL) - return EINVAL; - - if (ddp == NULL) - return EINVAL; - - idx_bits = 32 - tformat->sw_bits; - tformat->sw_bits = ddp->idx_bits; - tformat->rsvd_bits = ddp->idx_bits; - tformat->rsvd_shift = IPPOD_IDX_SHIFT; - tformat->rsvd_mask = (1 << tformat->rsvd_bits) - 1; - - CTR4(KTR_CXGBE, "tag format: sw %u, rsvd %u,%u, mask 0x%x.\n", - tformat->sw_bits, tformat->rsvd_bits, - tformat->rsvd_shift, tformat->rsvd_mask); - - *txsz = min(ULP2_MAX_PDU_PAYLOAD, - ddp->max_txsz - ISCSI_PDU_NONPAYLOAD_LEN); - *rxsz = min(ULP2_MAX_PDU_PAYLOAD, - ddp->max_rxsz - ISCSI_PDU_NONPAYLOAD_LEN); - CTR4(KTR_CXGBE, "max payload size: %u/%u, %u/%u.\n", - *txsz, ddp->max_txsz, *rxsz, ddp->max_rxsz); - return 0; -} - -/** - * cxgbei_ulp2_ddp_cleanup - release the cxgbX adapter's ddp resource - * @tdev: t4cdev adapter - * release all the resource held by the ddp pagepod manager for a given - * adapter if needed + * cxgbei_ddp_cleanup - release the adapter's ddp resources */ void -cxgbei_ulp2_ddp_cleanup(struct cxgbei_ulp2_ddp_info **ddp_pp) +cxgbei_ddp_cleanup(struct cxgbei_data *ci) { int i = 0; - struct cxgbei_ulp2_ddp_info *ddp = *ddp_pp; - if (ddp == NULL) - return; - - CTR2(KTR_CXGBE, "tdev, release ddp 0x%p, ref %d.\n", - ddp, atomic_load_acq_int(&ddp->refcnt)); - - if (ddp && (cxgbei_counter_dec_and_read(&ddp->refcnt) == 0)) { - *ddp_pp = NULL; - while (i < ddp->nppods) { - struct cxgbei_ulp2_gather_list *gl = ddp->gl_map[i]; - if (gl) { - int npods = (gl->nelem + IPPOD_PAGES_MAX - 1) - >> IPPOD_PAGES_SHIFT; - CTR2(KTR_CXGBE, - "tdev, ddp %d + %d.\n", i, npods); - free(gl, M_DEVBUF); - i += npods; - } else - i++; - } - bus_dmamap_unload(ddp->ulp_ddp_tag, ddp->ulp_ddp_map); - cxgbei_ulp2_free_big_mem(ddp); + while (i < ci->nppods) { + struct cxgbei_ulp2_gather_list *gl = ci->gl_map[i]; + if (gl) { + int npods = (gl->nelem + IPPOD_PAGES_MAX - 1) + >> IPPOD_PAGES_SHIFT; + free(gl, M_DEVBUF); + i += npods; + } else + i++; } -} - -/** - * ddp_init - initialize the cxgb3/4 adapter's ddp resource - * @tdev_name: device name - * @tdev: device - * @ddp: adapter's ddp info - * @uinfo: adapter's iscsi info - * initialize the ddp pagepod manager for a given adapter - */ -static void -ddp_init(void *tdev, - struct cxgbei_ulp2_ddp_info **ddp_pp, - struct ulp_iscsi_info *uinfo) -{ - struct cxgbei_ulp2_ddp_info *ddp = *ddp_pp; - unsigned int ppmax, bits; - int i, rc; - - if (uinfo->ulimit <= uinfo->llimit) { - printf("%s: tdev, ddp 0x%x >= 0x%x.\n", - __func__, uinfo->llimit, uinfo->ulimit); - return; - } - if (ddp) { - atomic_add_acq_int(&ddp->refcnt, 1); - CTR2(KTR_CXGBE, "tdev, ddp 0x%p already set up, %d.\n", - ddp, atomic_load_acq_int(&ddp->refcnt)); - return; - } - - ppmax = (uinfo->ulimit - uinfo->llimit + 1) >> IPPOD_SIZE_SHIFT; - if (ppmax <= 1024) { - CTR3(KTR_CXGBE, "tdev, ddp 0x%x ~ 0x%x, nppod %u < 1K.\n", - uinfo->llimit, uinfo->ulimit, ppmax); - return; - } - bits = (fls(ppmax) - 1) + 1; - - if (bits > IPPOD_IDX_MAX_SIZE) - bits = IPPOD_IDX_MAX_SIZE; - ppmax = (1 << (bits - 1)) - 1; - - ddp = cxgbei_ulp2_alloc_big_mem(sizeof(struct cxgbei_ulp2_ddp_info) + - ppmax * (sizeof(struct cxgbei_ulp2_gather_list *) + - sizeof(unsigned char))); - if (ddp == NULL) { - CTR1(KTR_CXGBE, "unable to alloc ddp 0x%d, ddp disabled.\n", - ppmax); - return; - } - ddp->colors = (unsigned char *)(ddp + 1); - ddp->gl_map = (struct cxgbei_ulp2_gather_list **)(ddp->colors + - ppmax * sizeof(unsigned char)); - *ddp_pp = ddp; - - mtx_init(&ddp->map_lock, "ddp lock", NULL, - MTX_DEF | MTX_DUPOK| MTX_RECURSE); - - atomic_set_acq_int(&ddp->refcnt, 1); - - /* dma_tag create */ - rc = ulp2_dma_tag_create(ddp); - if (rc) { - printf("%s: unable to alloc ddp 0x%d, ddp disabled.\n", - __func__, ppmax); - return; - } - - ddp->tdev = tdev; - ddp->max_txsz = min(uinfo->max_txsz, ULP2_MAX_PKT_SIZE); - ddp->max_rxsz = min(uinfo->max_rxsz, ULP2_MAX_PKT_SIZE); - ddp->llimit = uinfo->llimit; - ddp->ulimit = uinfo->ulimit; - ddp->nppods = ppmax; - ddp->idx_last = ppmax; - ddp->idx_bits = bits; - ddp->idx_mask = (1 << bits) - 1; - ddp->rsvd_tag_mask = (1 << (bits + IPPOD_IDX_SHIFT)) - 1; - - CTR2(KTR_CXGBE, - "gl map 0x%p, idx_last %u.\n", ddp->gl_map, ddp->idx_last); - uinfo->tagmask = ddp->idx_mask << IPPOD_IDX_SHIFT; - for (i = 0; i < DDP_PGIDX_MAX; i++) - uinfo->pgsz_factor[i] = ddp_page_order[i]; - uinfo->ulimit = uinfo->llimit + (ppmax << IPPOD_SIZE_SHIFT); - - printf("nppods %u, bits %u, mask 0x%x,0x%x pkt %u/%u," - " %u/%u.\n", - ppmax, ddp->idx_bits, ddp->idx_mask, - ddp->rsvd_tag_mask, ddp->max_txsz, uinfo->max_txsz, - ddp->max_rxsz, uinfo->max_rxsz); - - rc = bus_dmamap_create(ddp->ulp_ddp_tag, 0, &ddp->ulp_ddp_map); - if (rc != 0) { - printf("%s: bus_dmamap_Create failed\n", __func__); - return; - } -} - -/** - * cxgbei_ulp2_ddp_init - initialize ddp functions - */ -void -cxgbei_ulp2_ddp_init(void *tdev, - struct cxgbei_ulp2_ddp_info **ddp_pp, - struct ulp_iscsi_info *uinfo) -{ - if (page_idx == DDP_PGIDX_MAX) { - page_idx = cxgbei_ulp2_ddp_find_page_index(PAGE_SIZE); - - if (page_idx == DDP_PGIDX_MAX) { - if (cxgbei_ulp2_ddp_adjust_page_table()) { - CTR1(KTR_CXGBE, "PAGE_SIZE %x, ddp disabled.\n", - PAGE_SIZE); - return; - } - } - page_idx = cxgbei_ulp2_ddp_find_page_index(PAGE_SIZE); - } - - ddp_init(tdev, ddp_pp, uinfo); + free(ci->colors, M_CXGBE); + free(ci->gl_map, M_CXGBE); } diff --git a/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.h b/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.h index 15726979554..7f7245bc87a 100644 --- a/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.h +++ b/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.h @@ -30,43 +30,6 @@ #ifndef __CXGBEI_ULP2_DDP_H__ #define __CXGBEI_ULP2_DDP_H__ -#include -#include -#include - -#include -#include -#include - -/* - * Structure used to return information to the iscsi layer. - */ -struct ulp_iscsi_info { - unsigned int offset; - unsigned int llimit; - unsigned int ulimit; - unsigned int tagmask; - unsigned char pgsz_factor[4]; - unsigned int max_rxsz; - unsigned int max_txsz; -}; - -/* - * struct cxgbei_ulp2_tag_format - cxgbei ulp tag format for an iscsi entity - * - * @sw_bits: # of bits used by iscsi software layer - * @rsvd_bits: # of bits used by h/w - * @rsvd_shift: h/w bits shift left - * @rsvd_mask: reserved bit mask - */ -typedef struct cxgbei_ulp2_tag_format { - unsigned char sw_bits; - unsigned char rsvd_bits; - unsigned char rsvd_shift; - unsigned char filler[1]; - uint32_t rsvd_mask; -}cxgbei_ulp2_tag_format; - #define CXGBEI_PAGE_MASK (~(PAGE_SIZE-1)) #define DDP_THRESHOLD 2048 @@ -90,7 +53,8 @@ typedef struct cxgbei_ulp2_tag_format { static inline int cxgbei_ulp2_is_ddp_tag(struct cxgbei_ulp2_tag_format *tformat, uint32_t tag) { - return !(tag & (1 << (tformat->rsvd_bits + tformat->rsvd_shift - 1))); + + return (!(tag & (1 << (tformat->rsvd_bits + tformat->rsvd_shift - 1)))); } /* @@ -102,9 +66,10 @@ cxgbei_ulp2_is_ddp_tag(struct cxgbei_ulp2_tag_format *tformat, uint32_t tag) */ static inline int cxgbei_ulp2_sw_tag_usable(struct cxgbei_ulp2_tag_format *tformat, - uint32_t sw_tag) + uint32_t sw_tag) { - return 1; + + return (1); /* XXXNP: huh? */ sw_tag >>= (32 - tformat->rsvd_bits + tformat->rsvd_shift); return !sw_tag; @@ -160,59 +125,6 @@ struct cxgbei_ulp2_gather_list { struct dma_segments dma_sg[0]; }; -struct cxgbei_ulp2_pagepod_hdr; -/* - * struct cxgbei_ulp2_ddp_info - direct data placement for pdu payload - * - * @list: list head to link elements - * @refcnt: count of iscsi entities using it - * @tdev: pointer to tXcdev used by cxgbX driver - * @max_txsz: max tx packet size for ddp - * @max_rxsz: max rx packet size for ddp - * @llimit: lower bound of the page pod memory - * @ulimit: upper bound of the page pod memory - * @nppods: # of page pod entries - * @idx_last: page pod entry last used - * @idx_bits: # of bits the pagepod index would take - * @idx_mask: pagepod index mask - * @rsvd_tag_mask: tag mask - * @map_lock: lock to synchonize access to the page pod map - * @gl_map: ddp memory gather list - */ -struct cxgbei_ulp2_ddp_info { - SLIST_ENTRY(cxgbei_ulp2_ddp_info) cxgbei_ulp2_ddp_list; - volatile int refcnt; - void *tdev; /* t5odev */ - unsigned int max_txsz; - unsigned int max_rxsz; - unsigned int llimit; - unsigned int ulimit; - unsigned int nppods; - unsigned int idx_last; - unsigned char idx_bits; - unsigned char filler[3]; - uint32_t idx_mask; - uint32_t rsvd_tag_mask; - bus_addr_t rsvd_page_phys_addr; - - int (*ddp_set_map)(struct cxgbei_ulp2_ddp_info *ddp, - void *isock, - struct cxgbei_ulp2_pagepod_hdr *hdr, - unsigned int idx, unsigned int npods, - struct cxgbei_ulp2_gather_list *gl, int reply); - void (*ddp_clear_map)(struct cxgbei_ulp2_ddp_info *ddp, - struct cxgbei_ulp2_gather_list *gl, - unsigned int tag, unsigned int idx, - unsigned int npods, - iscsi_socket *isock); - - struct mtx map_lock; - bus_dma_tag_t ulp_ddp_tag; - bus_dmamap_t ulp_ddp_map; - unsigned char *colors; - struct cxgbei_ulp2_gather_list **gl_map; -}; - #define IPPOD_SIZE sizeof(struct cxgbei_ulp2_pagepod) /* 64 */ #define IPPOD_SIZE_SHIFT 6 @@ -244,16 +156,15 @@ struct cxgbei_ulp2_ddp_info { #define V_IPPOD_PGSZ(x) ((x) << S_IPPOD_PGSZ) static inline uint32_t -cxgbei_ulp2_ddp_tag_base(unsigned int idx, struct cxgbei_ulp2_ddp_info *ddp, - struct cxgbei_ulp2_tag_format *tformat, uint32_t sw_tag) +cxgbei_ulp2_ddp_tag_base(u_int idx, u_char *colors, + struct cxgbei_ulp2_tag_format *tformat, uint32_t sw_tag) { - ddp->colors[idx]++; - if (ddp->colors[idx] == (1 << IPPOD_IDX_SHIFT)) - ddp->colors[idx] = 0; + if (__predict_false(++colors[idx] == 1 << IPPOD_IDX_SHIFT)) + colors[idx] = 0; - sw_tag <<= (tformat->rsvd_bits + tformat->rsvd_shift); + sw_tag <<= tformat->rsvd_bits + tformat->rsvd_shift; - return sw_tag | (idx << 6) | ddp->colors[idx]; + return (sw_tag | idx << IPPOD_IDX_SHIFT | colors[idx]); } #define ISCSI_PDU_NONPAYLOAD_LEN 312 /* bhs(48) + ahs(256) + digest(8) */ @@ -284,65 +195,20 @@ struct cxgbei_ulp2_pagepod { uint64_t addr[IPPOD_PAGES_MAX + 1]; }; -/* - * ddp page size array - */ -#define DDP_PGIDX_MAX 4 -extern unsigned char ddp_page_order[DDP_PGIDX_MAX]; -extern unsigned char page_idx; +int cxgbei_ulp2_ddp_tag_reserve(struct cxgbei_data *, void *, unsigned int, + struct cxgbei_ulp2_tag_format *, uint32_t *, + struct cxgbei_ulp2_gather_list *, int , int ); +void cxgbei_ulp2_ddp_tag_release(struct cxgbei_data *, uint32_t, + struct iscsi_socket *); +struct cxgbei_ulp2_gather_list *cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(u_int, + struct cxgbei_sgl *, u_int, struct cxgbei_data *, int); +void cxgbei_ulp2_ddp_release_gl(struct cxgbei_data *, + struct cxgbei_ulp2_gather_list *); -/* - * large memory chunk allocation/release - * use vmalloc() if kmalloc() fails - */ -static inline void * -cxgbei_ulp2_alloc_big_mem(unsigned int size) -{ - void *p = NULL; +int cxgbei_ulp2_ddp_find_page_index(u_long); +int cxgbei_ulp2_adapter_ddp_info(struct cxgbei_data *, + struct cxgbei_ulp2_tag_format *); - p = malloc(size, M_TEMP, M_NOWAIT | M_ZERO); - - return p; -} - -static inline void -cxgbei_ulp2_free_big_mem(void *addr) -{ - free(addr, M_TEMP); -} - -int cxgbei_ulp2_ddp_tag_reserve(struct cxgbei_ulp2_ddp_info *, - void *, unsigned int , - struct cxgbei_ulp2_tag_format *, uint32_t *, - struct cxgbei_ulp2_gather_list *, int , int ); -void cxgbei_ulp2_ddp_tag_release(struct cxgbei_ulp2_ddp_info *, - uint32_t, iscsi_socket *); - -struct cxgbei_ulp2_gather_list *cxgbei_ulp2_ddp_make_gl(unsigned int , - struct sglist *, - unsigned int , - struct pci_conf *, - int); - -struct cxgbei_ulp2_gather_list *cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec( - unsigned int, - cxgbei_sgl *, - unsigned int, - void *, - int); - -void cxgbei_ulp2_ddp_release_gl(struct cxgbei_ulp2_gather_list *, void *); - -int cxgbei_ulp2_ddp_find_page_index(unsigned long); -int cxgbei_ulp2_adapter_ddp_info(struct cxgbei_ulp2_ddp_info *, - struct cxgbei_ulp2_tag_format *, - unsigned int *, unsigned int *); - -void cxgbei_ulp2_ddp_cleanup(struct cxgbei_ulp2_ddp_info **); -void cxgbei_ulp2_ddp_init(void *, - struct cxgbei_ulp2_ddp_info **, - struct ulp_iscsi_info *); -int cxgbei_ulp2_init(void); -void cxgbei_ulp2_exit(void); +void cxgbei_ddp_cleanup(struct cxgbei_data *); #endif diff --git a/sys/dev/cxgbe/cxgbei/icl_cxgbei.c b/sys/dev/cxgbe/cxgbei/icl_cxgbei.c index 17f345c6ddc..00d8ba2c53a 100644 --- a/sys/dev/cxgbe/cxgbei/icl_cxgbei.c +++ b/sys/dev/cxgbe/cxgbei/icl_cxgbei.c @@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -446,6 +447,7 @@ icl_cxgbei_conn_free(struct icl_conn *ic) refcount_release(&icl_ncons); } +/* XXXNP: what is this for? There's no conn_start method. */ static int icl_conn_start(struct icl_conn *ic) { @@ -572,9 +574,8 @@ icl_cxgbei_conn_handoff(struct icl_conn *ic, int fd) ICL_CONN_UNLOCK(ic); error = icl_conn_start(ic); - if(!error) { - cxgbei_conn_set_ulp_mode(ic->ic_socket, ic); - } + if (!error) + cxgbei_conn_handoff(ic); return (error); } @@ -620,7 +621,7 @@ icl_cxgbei_conn_close(struct icl_conn *ic) //ICL_DEBUG("send/receive threads terminated"); ICL_CONN_UNLOCK(ic); - cxgbei_conn_close(ic->ic_socket); + cxgbei_conn_close(ic); soclose(ic->ic_socket); ICL_CONN_LOCK(ic); ic->ic_socket = NULL; @@ -690,6 +691,7 @@ icl_cxgbei_conn_task_setup(struct icl_conn *ic, struct ccb_scsiio *csio, void icl_cxgbei_conn_task_done(struct icl_conn *ic, void *prv) { + cxgbei_cleanup_task(ic, prv); uma_zfree(icl_transfer_zone, prv); } diff --git a/sys/dev/cxgbe/cxgbei/mbufq.h b/sys/dev/cxgbe/cxgbei/mbufq.h deleted file mode 100644 index 4d11bc88dbc..00000000000 --- a/sys/dev/cxgbe/cxgbei/mbufq.h +++ /dev/null @@ -1,121 +0,0 @@ -/************************************************************************** - -Copyright (c) 2007-2008, Chelsio Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Neither the name of the Chelsio Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -$FreeBSD$ - -***************************************************************************/ - -#ifndef CXGB_MBUFQ_H_ -#define CXGB_MBUFQ_H_ - -struct mbuf_head { - struct mbuf *head; - struct mbuf *tail; - uint32_t qlen; - uint32_t qsize; - struct mtx lock; -}; - -static __inline void -mbufq_init(struct mbuf_head *l) -{ - l->head = l->tail = NULL; - l->qlen = l->qsize = 0; -} - -static __inline int -mbufq_empty(struct mbuf_head *l) -{ - return (l->head == NULL); -} - -static __inline int -mbufq_len(struct mbuf_head *l) -{ - return (l->qlen); -} - -static __inline int -mbufq_size(struct mbuf_head *l) -{ - return (l->qsize); -} - -static __inline int -mbufq_head_size(struct mbuf_head *l) -{ - return (l->head ? l->head->m_pkthdr.len : 0); -} - -static __inline void -mbufq_tail(struct mbuf_head *l, struct mbuf *m) -{ - l->qlen++; - if (l->head == NULL) - l->head = m; - else - l->tail->m_nextpkt = m; - l->tail = m; - l->qsize += m->m_pkthdr.len; -} - -static __inline struct mbuf * -mbufq_dequeue(struct mbuf_head *l) -{ - struct mbuf *m; - - m = l->head; - if (m) { - if (m == l->tail) - l->head = l->tail = NULL; - else - l->head = m->m_nextpkt; - m->m_nextpkt = NULL; - l->qlen--; - l->qsize -= m->m_pkthdr.len; - } - - return (m); -} - -static __inline struct mbuf * -mbufq_peek(const struct mbuf_head *l) -{ - return (l->head); -} - -static __inline void -mbufq_append(struct mbuf_head *a, struct mbuf_head *b) -{ - if (a->tail) - a->tail->m_nextpkt = b->head; - if (b->tail) - a->tail = b->tail; - a->qlen += b->qlen; - a->qsize += b->qsize; -} -#endif /* CXGB_MBUFQ_H_ */ diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c index 68cd5576cde..ec69b6dd0c6 100644 --- a/sys/dev/cxgbe/tom/t4_cpl_io.c +++ b/sys/dev/cxgbe/tom/t4_cpl_io.c @@ -805,6 +805,10 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) close_conn(sc, toep); } +void (*cxgbei_fw4_ack)(struct toepcb *, int); +struct mbuf *(*cxgbei_writeq_len)(struct toepcb *, int *); +struct mbuf *(*cxgbei_writeq_next)(struct toepcb *); + /* Send ULP data over TOE using TX_DATA_WR. We send whole mbuf at once */ void t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop) @@ -840,7 +844,7 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop) if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) return; - sndptr = t4_queue_iscsi_callback(so, toep, 1, &qlen); + sndptr = cxgbei_writeq_len(toep, &qlen); if (!qlen) return; @@ -850,8 +854,7 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop) max_nsegs = max_dsgl_nsegs(tx_credits); if (drop) { - t4_cpl_iscsi_callback(toep->td, toep, &drop, - CPL_FW4_ACK); + cxgbei_fw4_ack(toep, drop); drop = 0; } @@ -951,7 +954,7 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop) tp->snd_max += ulp_len; /* goto next mbuf */ - sndptr = m = t4_queue_iscsi_callback(so, toep, 2, &qlen); + sndptr = m = cxgbei_writeq_next(toep); toep->flags |= TPF_TX_DATA_SENT; if (toep->tx_credits < MIN_OFLD_TX_CREDITS) { @@ -1268,91 +1271,6 @@ abort_status_to_errno(struct tcpcb *tp, unsigned int abort_reason) } } -int -cpl_not_handled(struct sge_iq *, const struct rss_header *, struct mbuf *); -/* - * tom_cpl_iscsi_callback - - * iscsi and tom would share the following cpl messages, so when any of these - * message is received, after tom is done with processing it, the messages - * needs to be forwarded to iscsi for further processing: - * - CPL_SET_TCB_RPL - * - CPL_RX_DATA_DDP - */ -void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *, void *, - unsigned int); - -struct mbuf *(*tom_queue_iscsi_callback)(struct socket *, unsigned int, int *); -/* - * Check if the handler function is set for a given CPL - * return 0 if the function is NULL or cpl_not_handled, 1 otherwise. - */ -int -t4tom_cpl_handler_registered(struct adapter *sc, unsigned int opcode) -{ - - MPASS(opcode < nitems(sc->cpl_handler)); - - return (sc->cpl_handler[opcode] && - sc->cpl_handler[opcode] != cpl_not_handled); -} - -/* - * set the tom_cpl_iscsi_callback function, this function should be used - * whenever both toe and iscsi need to process the same cpl msg. - */ -void -t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *, struct socket *, - void *, unsigned int)) -{ - - tom_cpl_iscsi_callback = fp; -} - -void -t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *, - unsigned int, int *qlen)) -{ - - tom_queue_iscsi_callback = fp; -} - -int -t4_cpl_iscsi_callback(struct tom_data *td, struct toepcb *toep, void *m, - unsigned int opcode) -{ - struct socket *so; - - if (opcode == CPL_FW4_ACK) - so = toep->inp->inp_socket; - else { - INP_WLOCK(toep->inp); - so = toep->inp->inp_socket; - INP_WUNLOCK(toep->inp); - } - - if (tom_cpl_iscsi_callback && so) { - if (toep->ulp_mode == ULP_MODE_ISCSI) { - tom_cpl_iscsi_callback(td, so, m, opcode); - return (0); - } - } - - return (1); -} - -struct mbuf * -t4_queue_iscsi_callback(struct socket *so, struct toepcb *toep, - unsigned int cmd, int *qlen) -{ - - if (tom_queue_iscsi_callback && so) { - if (toep->ulp_mode == ULP_MODE_ISCSI) - return (tom_queue_iscsi_callback(so, cmd, qlen)); - } - - return (NULL); -} - /* * TCP RST from the peer, timeout, or some other such critical error. */ @@ -1748,16 +1666,32 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) t4_push_frames(sc, toep, plen); } else if (plen > 0) { struct sockbuf *sb = &so->so_snd; + int sbu; - if (toep->ulp_mode == ULP_MODE_ISCSI) - t4_cpl_iscsi_callback(toep->td, toep, &plen, - CPL_FW4_ACK); - else { - SOCKBUF_LOCK(sb); + SOCKBUF_LOCK(sb); + sbu = sbused(sb); + if (toep->ulp_mode == ULP_MODE_ISCSI) { + + if (__predict_false(sbu > 0)) { + /* + * The data trasmitted before the tid's ULP mode + * changed to ISCSI is still in so_snd. + * Incoming credits should account for so_snd + * first. + */ + sbdrop_locked(sb, min(sbu, plen)); + plen -= min(sbu, plen); + } + /* XXXNP: sowwakeup_locked causes a LOR. */ + SOCKBUF_UNLOCK(sb); + + if (__predict_true(plen > 0)) + cxgbei_fw4_ack(toep, plen); + } else { sbdrop_locked(sb, plen); - sowwakeup_locked(so); - SOCKBUF_UNLOCK_ASSERT(sb); + sowwakeup_locked(so); /* unlocks so_snd */ } + SOCKBUF_UNLOCK_ASSERT(sb); } INP_WUNLOCK(inp); @@ -1781,14 +1715,21 @@ do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) if (is_ftid(sc, tid)) return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */ - else { - struct toepcb *toep = lookup_tid(sc, tid); - t4_cpl_iscsi_callback(toep->td, toep, m, CPL_SET_TCB_RPL); - return (0); - } + /* + * TOM and/or other ULPs don't request replies for CPL_SET_TCB or + * CPL_SET_TCB_FIELD requests. This can easily change and when it does + * the dispatch code will go here. + */ +#ifdef INVARIANTS + panic("%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p", __func__, + tid, iq); +#else + log(LOG_ERR, "%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p\n", + __func__, tid, iq); +#endif - CXGBE_UNIMPLEMENTED(__func__); + return (0); } void diff --git a/sys/dev/cxgbe/tom/t4_ddp.c b/sys/dev/cxgbe/tom/t4_ddp.c index c4ebfb68719..2554b94e09d 100644 --- a/sys/dev/cxgbe/tom/t4_ddp.c +++ b/sys/dev/cxgbe/tom/t4_ddp.c @@ -437,6 +437,8 @@ wakeup: F_DDP_INVALID_TAG | F_DDP_COLOR_ERR | F_DDP_TID_MISMATCH |\ F_DDP_INVALID_PPOD | F_DDP_HDRCRC_ERR | F_DDP_DATACRC_ERR) +void (*cxgbei_rx_data_ddp)(struct toepcb *, const struct cpl_rx_data_ddp *); + static int do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { @@ -445,7 +447,6 @@ do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) unsigned int tid = GET_TID(cpl); uint32_t vld; struct toepcb *toep = lookup_tid(sc, tid); - struct tom_data *td = toep->td; KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); @@ -457,10 +458,10 @@ do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) panic("%s: DDP error 0x%x (tid %d, toep %p)", __func__, vld, tid, toep); } + if (toep->ulp_mode == ULP_MODE_ISCSI) { - if (!t4_cpl_iscsi_callback(td, toep, (void *)cpl, - CPL_RX_DATA_DDP)) - return (0); + cxgbei_rx_data_ddp(toep, cpl); + return (0); } handle_ddp_data(toep, cpl->u.ddp_report, cpl->seq, be16toh(cpl->len)); diff --git a/sys/dev/cxgbe/tom/t4_tom.h b/sys/dev/cxgbe/tom/t4_tom.h index c54f6be2494..cfff4644850 100644 --- a/sys/dev/cxgbe/tom/t4_tom.h +++ b/sys/dev/cxgbe/tom/t4_tom.h @@ -115,6 +115,7 @@ struct toepcb { int rx_credits; /* rx credits (in bytes) to be returned to hw */ u_int ulp_mode; /* ULP mode */ + void *ulpcb; u_int ddp_flags; struct ddp_buffer *db[2]; @@ -288,17 +289,5 @@ void insert_ddp_data(struct toepcb *, uint32_t); /* ULP related */ #define CXGBE_ISCSI_MBUF_TAG 50 -int t4tom_cpl_handler_registered(struct adapter *, unsigned int); -void t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *, - struct socket *, void *, unsigned int)); -void t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *, - unsigned int, int *)); void t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int); -int t4_cpl_iscsi_callback(struct tom_data *, struct toepcb *, void *, uint32_t); -struct mbuf *t4_queue_iscsi_callback(struct socket *, struct toepcb *, uint32_t, - int *); -extern void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *, - void *, unsigned int); -extern struct mbuf *(*tom_queue_iscsi_callback)(struct socket*, unsigned int, - int *); #endif