cxgbei: Support unmapped I/O requests.

- Add icl_pdu_append_bio and icl_pdu_get_bio methods. - Add new page pod routines for allocating and writing page pods for unmapped bio requests. Use these new routines for setting up DDP for iSCSI tasks with a SCSI I/O CCB which uses CAM_DATA_BIO. - When ICL_NOCOPY is used to append data from an unmapped I/O request to a PDU, construct unmapped mbufs from the relevant pages backing the struct bio. This also requires changes in the t4_push_pdus path to support unmapped mbufs. Sponsored by: Chelsio Communications Differential Revision: https://reviews.freebsd.org/D34383
2026-06-04 22:32:43 -04:00 · 2022-03-10 15:50:52 -08:00 · 2022-03-10 15:50:52 -08:00 · 2beaefe884
commit 2beaefe884
parent 530e725d8e
4 changed files with 416 additions and 76 deletions
--- a/sys/dev/cxgbe/cxgbei/icl_cxgbei.c
+++ b/sys/dev/cxgbe/cxgbei/icl_cxgbei.c
@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");

 #ifdef TCP_OFFLOAD
 #include <sys/param.h>
+#include <sys/bio.h>
 #include <sys/capsicum.h>
 #include <sys/condvar.h>
 #include <sys/conf.h>
@ -61,6 +62,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/uio.h>
 #include <machine/bus.h>
 #include <vm/vm.h>
+#include <vm/vm_page.h>
 #include <vm/pmap.h>
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
@ -132,7 +134,9 @@ static volatile u_int icl_cxgbei_ncons;
 static icl_conn_new_pdu_t	icl_cxgbei_conn_new_pdu;
 static icl_conn_pdu_data_segment_length_t
 				    icl_cxgbei_conn_pdu_data_segment_length;
+static icl_conn_pdu_append_bio_t	icl_cxgbei_conn_pdu_append_bio;
 static icl_conn_pdu_append_data_t	icl_cxgbei_conn_pdu_append_data;
+static icl_conn_pdu_get_bio_t	icl_cxgbei_conn_pdu_get_bio;
 static icl_conn_pdu_get_data_t	icl_cxgbei_conn_pdu_get_data;
 static icl_conn_pdu_queue_t	icl_cxgbei_conn_pdu_queue;
 static icl_conn_pdu_queue_cb_t	icl_cxgbei_conn_pdu_queue_cb;
@ -149,7 +153,9 @@ static kobj_method_t icl_cxgbei_methods[] = {
 	KOBJMETHOD(icl_conn_pdu_free, icl_cxgbei_conn_pdu_free),
 	KOBJMETHOD(icl_conn_pdu_data_segment_length,
 	    icl_cxgbei_conn_pdu_data_segment_length),
+	KOBJMETHOD(icl_conn_pdu_append_bio, icl_cxgbei_conn_pdu_append_bio),
 	KOBJMETHOD(icl_conn_pdu_append_data, icl_cxgbei_conn_pdu_append_data),
+	KOBJMETHOD(icl_conn_pdu_get_bio, icl_cxgbei_conn_pdu_get_bio),
 	KOBJMETHOD(icl_conn_pdu_get_data, icl_cxgbei_conn_pdu_get_data),
 	KOBJMETHOD(icl_conn_pdu_queue, icl_cxgbei_conn_pdu_queue),
 	KOBJMETHOD(icl_conn_pdu_queue_cb, icl_cxgbei_conn_pdu_queue_cb),
@ -552,6 +558,189 @@ out:
 	kthread_exit();
 }

+static void
+cxgbei_free_mext_pg(struct mbuf *m)
+{
+	struct icl_cxgbei_pdu *icp;
+
+	M_ASSERTEXTPG(m);
+
+	/*
+	 * Nothing to do for the pages; they are owned by the PDU /
+	 * I/O request.
+	 */
+
+	/* Drop reference on the PDU. */
+	icp = m->m_ext.ext_arg1;
+	if (atomic_fetchadd_int(&icp->ref_cnt, -1) == 1)
+		icl_cxgbei_pdu_call_cb(&icp->ip);
+}
+
+static struct mbuf *
+cxgbei_getm(size_t len, int flags)
+{
+	struct mbuf *m, *m0, *m_tail;
+
+	m_tail = m0 = NULL;
+
+	/* Allocate as jumbo mbufs of size MJUM16BYTES. */
+	while (len >= MJUM16BYTES) {
+		m = m_getjcl(M_NOWAIT, MT_DATA, 0, MJUM16BYTES);
+		if (__predict_false(m == NULL)) {
+			if ((flags & M_WAITOK) != 0) {
+				/* Fall back to non-jumbo mbufs. */
+				break;
+			}
+			return (NULL);
+		}
+		if (m0 == NULL) {
+			m0 = m_tail = m;
+		} else {
+			m_tail->m_next = m;
+			m_tail = m;
+		}
+		len -= MJUM16BYTES;
+	}
+
+	/* Allocate mbuf chain for the remaining data. */
+	if (len != 0) {
+		m = m_getm2(NULL, len, flags, MT_DATA, 0);
+		if (__predict_false(m == NULL)) {
+			m_freem(m0);
+			return (NULL);
+		}
+		if (m0 == NULL)
+			m0 = m;
+		else
+			m_tail->m_next = m;
+	}
+
+	return (m0);
+}
+
+int
+icl_cxgbei_conn_pdu_append_bio(struct icl_conn *ic, struct icl_pdu *ip,
+    struct bio *bp, size_t offset, size_t len, int flags)
+{
+	struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
+	struct mbuf *m, *m_tail;
+	vm_offset_t vaddr;
+	size_t page_offset, todo, mtodo;
+	boolean_t mapped;
+	int i;
+
+	MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE);
+	MPASS(ic == ip->ip_conn);
+	KASSERT(len > 0, ("%s: len is %jd", __func__, (intmax_t)len));
+
+	m_tail = ip->ip_data_mbuf;
+	if (m_tail != NULL)
+		for (; m_tail->m_next != NULL; m_tail = m_tail->m_next)
+			;
+
+	MPASS(bp->bio_flags & BIO_UNMAPPED);
+	if (offset < PAGE_SIZE - bp->bio_ma_offset) {
+		page_offset = bp->bio_ma_offset + offset;
+		i = 0;
+	} else {
+		offset -= PAGE_SIZE - bp->bio_ma_offset;
+		for (i = 1; offset >= PAGE_SIZE; i++)
+			offset -= PAGE_SIZE;
+		page_offset = offset;
+	}
+
+	if (flags & ICL_NOCOPY) {
+		m = NULL;
+		while (len > 0) {
+			if (m == NULL) {
+				m = mb_alloc_ext_pgs(flags & ~ICL_NOCOPY,
+				    cxgbei_free_mext_pg);
+				if (__predict_false(m == NULL))
+					return (ENOMEM);
+				atomic_add_int(&icp->ref_cnt, 1);
+				m->m_ext.ext_arg1 = icp;
+				m->m_epg_1st_off = page_offset;
+			}
+
+			todo = MIN(len, PAGE_SIZE - page_offset);
+
+			m->m_epg_pa[m->m_epg_npgs] =
+			    VM_PAGE_TO_PHYS(bp->bio_ma[i]);
+			m->m_epg_npgs++;
+			m->m_epg_last_len = todo;
+			m->m_len += todo;
+			m->m_ext.ext_size += PAGE_SIZE;
+			MBUF_EXT_PGS_ASSERT_SANITY(m);
+
+			if (m->m_epg_npgs == MBUF_PEXT_MAX_PGS) {
+				if (m_tail != NULL)
+					m_tail->m_next = m;
+				else
+					ip->ip_data_mbuf = m;
+				m_tail = m;
+				ip->ip_data_len += m->m_len;
+				m = NULL;
+			}
+
+			page_offset = 0;
+			len -= todo;
+			i++;
+		}
+
+		if (m != NULL) {
+			if (m_tail != NULL)
+				m_tail->m_next = m;
+			else
+				ip->ip_data_mbuf = m;
+			ip->ip_data_len += m->m_len;
+		}
+		return (0);
+	}
+
+	m = cxgbei_getm(len, flags);
+	if (__predict_false(m == NULL))
+		return (ENOMEM);
+
+	if (ip->ip_data_mbuf == NULL) {
+		ip->ip_data_mbuf = m;
+		ip->ip_data_len = len;
+	} else {
+		m_tail->m_next = m;
+		ip->ip_data_len += len;
+	}
+
+	while (len > 0) {
+		todo = MIN(len, PAGE_SIZE - page_offset);
+
+		mapped = pmap_map_io_transient(bp->bio_ma + i, &vaddr, 1,
+		    FALSE);
+
+		do {
+			mtodo = min(todo, M_SIZE(m) - m->m_len);
+			memcpy(mtod(m, char *) + m->m_len, (char *)vaddr +
+			    page_offset, mtodo);
+			m->m_len += mtodo;
+			if (m->m_len == M_SIZE(m))
+				m = m->m_next;
+			page_offset += mtodo;
+			todo -= mtodo;
+		} while (todo > 0);
+
+		if (__predict_false(mapped))
+			pmap_unmap_io_transient(bp->bio_ma + 1, &vaddr, 1,
+			    FALSE);
+
+		page_offset = 0;
+		len -= todo;
+		i++;
+	}
+
+	MPASS(ip->ip_data_len <= max(ic->ic_max_send_data_segment_length,
+	    ic->ic_hw_isomax));
+
+	return (0);
+}
+
 int
 icl_cxgbei_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *ip,
    const void *addr, size_t len, int flags)
@ -592,58 +781,74 @@ icl_cxgbei_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *ip,
 		return (0);
 	}

+	m = cxgbei_getm(len, flags);
+	if (__predict_false(m == NULL))
+		return (ENOMEM);
+
+	if (ip->ip_data_mbuf == NULL) {
+		ip->ip_data_mbuf = m;
+		ip->ip_data_len = len;
+	} else {
+		m_tail->m_next = m;
+		ip->ip_data_len += len;
+	}
 	src = (const char *)addr;
-
-	/* Allocate as jumbo mbufs of size MJUM16BYTES. */
-	while (len >= MJUM16BYTES) {
-		m = m_getjcl(M_NOWAIT, MT_DATA, 0, MJUM16BYTES);
-		if (__predict_false(m == NULL)) {
-			if ((flags & M_WAITOK) != 0) {
-				/* Fall back to non-jumbo mbufs. */
-				break;
-			}
-			return (ENOMEM);
-		}
-		memcpy(mtod(m, void *), src, MJUM16BYTES);
-		m->m_len = MJUM16BYTES;
-		if (ip->ip_data_mbuf == NULL) {
-			ip->ip_data_mbuf = m_tail = m;
-			ip->ip_data_len = MJUM16BYTES;
-		} else {
-			m_tail->m_next = m;
-			m_tail = m_tail->m_next;
-			ip->ip_data_len += MJUM16BYTES;
-		}
-		src += MJUM16BYTES;
-		len -= MJUM16BYTES;
+	for (; m != NULL; m = m->m_next) {
+		m->m_len = min(len, M_SIZE(m));
+		memcpy(mtod(m, void *), src, m->m_len);
+		src += m->m_len;
+		len -= m->m_len;
 	}
+	MPASS(len == 0);

-	/* Allocate mbuf chain for the remaining data. */
-	if (len != 0) {
-		m = m_getm2(NULL, len, flags, MT_DATA, 0);
-		if (__predict_false(m == NULL))
-			return (ENOMEM);
-		if (ip->ip_data_mbuf == NULL) {
-			ip->ip_data_mbuf = m;
-			ip->ip_data_len = len;
-		} else {
-			m_tail->m_next = m;
-			ip->ip_data_len += len;
-		}
-		for (; m != NULL; m = m->m_next) {
-			m->m_len = min(len, M_SIZE(m));
-			memcpy(mtod(m, void *), src, m->m_len);
-			src += m->m_len;
-			len -= m->m_len;
-		}
-		MPASS(len == 0);
-	}
 	MPASS(ip->ip_data_len <= max(ic->ic_max_send_data_segment_length,
 	    ic->ic_hw_isomax));

 	return (0);
 }

+void
+icl_cxgbei_conn_pdu_get_bio(struct icl_conn *ic, struct icl_pdu *ip,
+    size_t pdu_off, struct bio *bp, size_t bio_off, size_t len)
+{
+	struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
+	vm_offset_t vaddr;
+	size_t page_offset, todo;
+	boolean_t mapped;
+	int i;
+
+	if (icp->icp_flags & ICPF_RX_DDP)
+		return; /* data is DDP'ed, no need to copy */
+
+	MPASS(bp->bio_flags & BIO_UNMAPPED);
+	if (bio_off < PAGE_SIZE - bp->bio_ma_offset) {
+		page_offset = bp->bio_ma_offset + bio_off;
+		i = 0;
+	} else {
+		bio_off -= PAGE_SIZE - bp->bio_ma_offset;
+		for (i = 1; bio_off >= PAGE_SIZE; i++)
+			bio_off -= PAGE_SIZE;
+		page_offset = bio_off;
+	}
+
+	while (len > 0) {
+		todo = MIN(len, PAGE_SIZE - page_offset);
+
+		mapped = pmap_map_io_transient(bp->bio_ma + i, &vaddr, 1,
+		    FALSE);
+		m_copydata(ip->ip_data_mbuf, pdu_off, todo, (char *)vaddr +
+		    page_offset);
+		if (__predict_false(mapped))
+			pmap_unmap_io_transient(bp->bio_ma + 1, &vaddr, 1,
+			    FALSE);
+
+		page_offset = 0;
+		pdu_off += todo;
+		len -= todo;
+		i++;
+	}
+}
+
 void
 icl_cxgbei_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip,
    size_t off, void *addr, size_t len)
@ -716,7 +921,7 @@ icl_cxgbei_new_conn(const char *name, struct mtx *lock)
 #endif
 	ic->ic_name = name;
 	ic->ic_offload = "cxgbei";
-	ic->ic_unmapped = false;
+	ic->ic_unmapped = true;

 	CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc);

@ -1251,22 +1456,45 @@ no_ddp:
 	}
 	prsv = &ddp->prsv;

-	/* XXX add support for all CAM_DATA_ types */
-	MPASS((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR);
-	rc = t4_alloc_page_pods_for_buf(pr, (vm_offset_t)csio->data_ptr,
-	    csio->dxfer_len, prsv);
-	if (rc != 0) {
-		free(ddp, M_CXGBEI);
-		goto no_ddp;
-	}
-
 	mbufq_init(&mq, INT_MAX);
-	rc = t4_write_page_pods_for_buf(sc, toep, prsv,
-	    (vm_offset_t)csio->data_ptr, csio->dxfer_len, &mq);
-	if (__predict_false(rc != 0)) {
-		mbufq_drain(&mq);
-		t4_free_page_pods(prsv);
+	switch (csio->ccb_h.flags & CAM_DATA_MASK) {
+	case CAM_DATA_BIO:
+		rc = t4_alloc_page_pods_for_bio(pr,
+		    (struct bio *)csio->data_ptr, prsv);
+		if (rc != 0) {
+			free(ddp, M_CXGBEI);
+			goto no_ddp;
+		}
+
+		rc = t4_write_page_pods_for_bio(sc, toep, prsv,
+		    (struct bio *)csio->data_ptr, &mq);
+		if (__predict_false(rc != 0)) {
+			mbufq_drain(&mq);
+			t4_free_page_pods(prsv);
+			free(ddp, M_CXGBEI);
+			goto no_ddp;
+		}
+		break;
+	case CAM_DATA_VADDR:
+		rc = t4_alloc_page_pods_for_buf(pr, (vm_offset_t)csio->data_ptr,
+		    csio->dxfer_len, prsv);
+		if (rc != 0) {
+			free(ddp, M_CXGBEI);
+			goto no_ddp;
+		}
+
+		rc = t4_write_page_pods_for_buf(sc, toep, prsv,
+		    (vm_offset_t)csio->data_ptr, csio->dxfer_len, &mq);
+		if (__predict_false(rc != 0)) {
+			mbufq_drain(&mq);
+			t4_free_page_pods(prsv);
+			free(ddp, M_CXGBEI);
+			goto no_ddp;
+		}
+		break;
+	default:
 		free(ddp, M_CXGBEI);
+		rc = EINVAL;
 		goto no_ddp;
 	}

--- a/sys/dev/cxgbe/tom/t4_cpl_io.c
+++ b/sys/dev/cxgbe/tom/t4_cpl_io.c
@ -1002,7 +1002,7 @@ write_iscsi_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr)
 	int tx_credits, shove, npdu, wr_len;
 	uint16_t iso_mss;
 	static const u_int ulp_extra_len[] = {0, 4, 4, 8};
-	bool iso;
+	bool iso, nomap_mbuf_seen;

 	M_ASSERTPKTHDR(sndptr);

@ -1030,8 +1030,15 @@ write_iscsi_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr)
 	plen = 0;
 	nsegs = 0;
 	max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
+	nomap_mbuf_seen = false;
 	for (m = sndptr; m != NULL; m = m->m_next) {
-		int n = sglist_count(mtod(m, void *), m->m_len);
+		int n;
+
+		if (m->m_flags & M_EXTPG)
+			n = sglist_count_mbuf_epg(m, mtod(m, vm_offset_t),
+			    m->m_len);
+		else
+			n = sglist_count(mtod(m, void *), m->m_len);

 		nsegs += n;
 		plen += m->m_len;
@ -1040,9 +1047,11 @@ write_iscsi_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr)
 		 * This mbuf would send us _over_ the nsegs limit.
 		 * Suspend tx because the PDU can't be sent out.
 		 */
-		if (plen > max_imm && nsegs > max_nsegs)
+		if ((nomap_mbuf_seen || plen > max_imm) && nsegs > max_nsegs)
 			return (NULL);

+		if (m->m_flags & M_EXTPG)
+			nomap_mbuf_seen = true;
 		if (max_nsegs_1mbuf < n)
 			max_nsegs_1mbuf = n;
 	}
@ -1075,7 +1084,7 @@ write_iscsi_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr)
 	wr_len = sizeof(*txwr);
 	if (iso)
 		wr_len += sizeof(struct cpl_tx_data_iso);
-	if (plen <= max_imm) {
+	if (plen <= max_imm && !nomap_mbuf_seen) {
 		/* Immediate data tx */
 		imm_data = plen;
 		wr_len += plen;
--- a/sys/dev/cxgbe/tom/t4_ddp.c
+++ b/sys/dev/cxgbe/tom/t4_ddp.c
@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$");

 #include <sys/param.h>
 #include <sys/aio.h>
+#include <sys/bio.h>
 #include <sys/file.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
@ -887,14 +888,11 @@ alloc_page_pods(struct ppod_region *pr, u_int nppods, u_int pgsz_idx,
 	return (0);
 }

-int
-t4_alloc_page_pods_for_ps(struct ppod_region *pr, struct pageset *ps)
+static int
+t4_alloc_page_pods_for_vmpages(struct ppod_region *pr, vm_page_t *pages,
+    int npages, struct ppod_reservation *prsv)
 {
 	int i, hcf, seglen, idx, nppods;
-	struct ppod_reservation *prsv = &ps->prsv;
-
-	KASSERT(prsv->prsv_nppods == 0,
-	    ("%s: page pods already allocated", __func__));

 	/*
 	 * The DDP page size is unrelated to the VM page size.  We combine
@ -904,11 +902,11 @@ t4_alloc_page_pods_for_ps(struct ppod_region *pr, struct pageset *ps)
 	 * the page list.
 	 */
 	hcf = 0;
-	for (i = 0; i < ps->npages; i++) {
+	for (i = 0; i < npages; i++) {
 		seglen = PAGE_SIZE;
-		while (i < ps->npages - 1 &&
-		    VM_PAGE_TO_PHYS(ps->pages[i]) + PAGE_SIZE ==
-		    VM_PAGE_TO_PHYS(ps->pages[i + 1])) {
+		while (i < npages - 1 &&
+		    VM_PAGE_TO_PHYS(pages[i]) + PAGE_SIZE ==
+		    VM_PAGE_TO_PHYS(pages[i + 1])) {
 			seglen += PAGE_SIZE;
 			i++;
 		}
@ -931,12 +929,35 @@ t4_alloc_page_pods_for_ps(struct ppod_region *pr, struct pageset *ps)
 have_pgsz:
 	MPASS(idx <= M_PPOD_PGSZ);

-	nppods = pages_to_nppods(ps->npages, pr->pr_page_shift[idx]);
+	nppods = pages_to_nppods(npages, pr->pr_page_shift[idx]);
 	if (alloc_page_pods(pr, nppods, idx, prsv) != 0)
-		return (0);
+		return (ENOMEM);
 	MPASS(prsv->prsv_nppods > 0);

-	return (1);
+	return (0);
+}
+
+int
+t4_alloc_page_pods_for_ps(struct ppod_region *pr, struct pageset *ps)
+{
+	struct ppod_reservation *prsv = &ps->prsv;
+
+	KASSERT(prsv->prsv_nppods == 0,
+	    ("%s: page pods already allocated", __func__));
+
+	return (t4_alloc_page_pods_for_vmpages(pr, ps->pages, ps->npages,
+	    prsv));
+}
+
+int
+t4_alloc_page_pods_for_bio(struct ppod_region *pr, struct bio *bp,
+    struct ppod_reservation *prsv)
+{
+
+	MPASS(bp->bio_flags & BIO_UNMAPPED);
+
+	return (t4_alloc_page_pods_for_vmpages(pr, bp->bio_ma, bp->bio_ma_n,
+	    prsv));
 }

 int
@ -1189,6 +1210,83 @@ alloc_raw_wr_mbuf(int len)
 	return (m);
 }

+int
+t4_write_page_pods_for_bio(struct adapter *sc, struct toepcb *toep,
+    struct ppod_reservation *prsv, struct bio *bp, struct mbufq *wrq)
+{
+	struct ulp_mem_io *ulpmc;
+	struct ulptx_idata *ulpsc;
+	struct pagepod *ppod;
+	int i, j, k, n, chunk, len, ddp_pgsz, idx;
+	u_int ppod_addr;
+	uint32_t cmd;
+	struct ppod_region *pr = prsv->prsv_pr;
+	vm_paddr_t pa;
+	struct mbuf *m;
+
+	MPASS(bp->bio_flags & BIO_UNMAPPED);
+
+	cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE));
+	if (is_t4(sc))
+		cmd |= htobe32(F_ULP_MEMIO_ORDER);
+	else
+		cmd |= htobe32(F_T5_ULP_MEMIO_IMM);
+	ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)];
+	ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask);
+	for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) {
+
+		/* How many page pods are we writing in this cycle */
+		n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS);
+		MPASS(n > 0);
+		chunk = PPOD_SZ(n);
+		len = roundup2(sizeof(*ulpmc) + sizeof(*ulpsc) + chunk, 16);
+
+		m = alloc_raw_wr_mbuf(len);
+		if (m == NULL)
+			return (ENOMEM);
+
+		ulpmc = mtod(m, struct ulp_mem_io *);
+		INIT_ULPTX_WR(ulpmc, len, 0, toep->tid);
+		ulpmc->cmd = cmd;
+		ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32));
+		ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16));
+		ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5));
+
+		ulpsc = (struct ulptx_idata *)(ulpmc + 1);
+		ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
+		ulpsc->len = htobe32(chunk);
+
+		ppod = (struct pagepod *)(ulpsc + 1);
+		for (j = 0; j < n; i++, j++, ppod++) {
+			ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID |
+			    V_PPOD_TID(toep->tid) |
+			    (prsv->prsv_tag & ~V_PPOD_PGSZ(M_PPOD_PGSZ)));
+			ppod->len_offset = htobe64(V_PPOD_LEN(bp->bio_bcount) |
+			    V_PPOD_OFST(bp->bio_ma_offset));
+			ppod->rsvd = 0;
+			idx = i * PPOD_PAGES * (ddp_pgsz / PAGE_SIZE);
+			for (k = 0; k < nitems(ppod->addr); k++) {
+				if (idx < bp->bio_ma_n) {
+					pa = VM_PAGE_TO_PHYS(bp->bio_ma[idx]);
+					ppod->addr[k] = htobe64(pa);
+					idx += ddp_pgsz / PAGE_SIZE;
+				} else
+					ppod->addr[k] = 0;
+#if 0
+				CTR5(KTR_CXGBE,
+				    "%s: tid %d ppod[%d]->addr[%d] = %p",
+				    __func__, toep->tid, i, k,
+				    be64toh(ppod->addr[k]));
+#endif
+			}
+		}
+
+		mbufq_enqueue(wrq, m);
+	}
+
+	return (0);
+}
+
 int
 t4_write_page_pods_for_buf(struct adapter *sc, struct toepcb *toep,
    struct ppod_reservation *prsv, vm_offset_t buf, int buflen,
@ -1398,7 +1496,7 @@ prep_pageset(struct adapter *sc, struct toepcb *toep, struct pageset *ps)
 	struct tom_data *td = sc->tom_softc;

 	if (ps->prsv.prsv_nppods == 0 &&
-	    !t4_alloc_page_pods_for_ps(&td->pr, ps)) {
+	    t4_alloc_page_pods_for_ps(&td->pr, ps) != 0) {
 		return (0);
 	}
 	if (!(ps->flags & PS_PPODS_WRITTEN) &&
--- a/sys/dev/cxgbe/tom/t4_tom.h
+++ b/sys/dev/cxgbe/tom/t4_tom.h
@ -89,6 +89,7 @@ enum {
 	DDP_DEAD	= (1 << 6),	/* toepcb is shutting down */
 };

+struct bio;
 struct ctl_sg_entry;
 struct sockopt;
 struct offload_settings;
@ -488,12 +489,16 @@ int t4_init_ppod_region(struct ppod_region *, struct t4_range *, u_int,
    const char *);
 void t4_free_ppod_region(struct ppod_region *);
 int t4_alloc_page_pods_for_ps(struct ppod_region *, struct pageset *);
+int t4_alloc_page_pods_for_bio(struct ppod_region *, struct bio *,
+    struct ppod_reservation *);
 int t4_alloc_page_pods_for_buf(struct ppod_region *, vm_offset_t, int,
    struct ppod_reservation *);
 int t4_alloc_page_pods_for_sgl(struct ppod_region *, struct ctl_sg_entry *, int,
    struct ppod_reservation *);
 int t4_write_page_pods_for_ps(struct adapter *, struct sge_wrq *, int,
    struct pageset *);
+int t4_write_page_pods_for_bio(struct adapter *, struct toepcb *,
+    struct ppod_reservation *, struct bio *, struct mbufq *);
 int t4_write_page_pods_for_buf(struct adapter *, struct toepcb *,
    struct ppod_reservation *, vm_offset_t, int, struct mbufq *);
 int t4_write_page_pods_for_sgl(struct adapter *, struct toepcb *,