upstream/opnsense-src
1
0
Fork 0
mirror of https://github.com/opnsense/src.git synced 2026-06-11 01:30:30 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

First round of code cleanup and reorganization. This is mostly code on

Browse source 
the slow path.  Changes to the fast path will follow later.

- Use the standard ULD registration and activation mechanism offered by
  if_cxgbe.  This eliminates all the code that managed the list of
  offload_device structures.  This simplifies the CPL dispatch too by
  eliminating t4tom_cpl_handler_register_flag and associated code.

- Remove all unused or write-only fields from various structures
  (iscsi_socket, offload_device, cxgbei_ulp2_ddp_info, ulp_iscsi_info)

- Eliminate the two line wrappers around malloc/free.  While here,
  switch to using M_CXGBE for all allocations.

- Simplify the page size settings in the chip (for iSCSI).  This ULD
  "owns" these settings so it should simply write the values that it
  wants to the A_ULP_RX_ISCSI_TAGMASK and A_ULP_RX_ISCSI_PSZ registers.
  This eliminates the globals ddp_page_order[], ddp_page_shift[],
  page_idx and all related code.

- Maintain the per-adapter ULD state in one data structure instead of
  two.  This consolidates struct offload_device and struct
  cxgbei_ulp2_ddp_info and into struct cxgbei_data, which is stored in
  adapter->iscsi_softc.

- Leave socket->so_emuldata alone, it exists for a different purpose
  (which is definitely not iSCSI).  Store the per-socket offload state
  in a new field in struct icl_conn instead.  (The new field exists only
  in this project branch and hasn't been reviewed for inclusion into
  head yet).

- Switch to the system version of mbufq.

- Tidy up the CPL/callback dispatch from if_cxgbe/t4_tom into cxgbei.
  The tid/toepcb is always available (t4_tom has looked it up) so it's a
  waste of time looking it up again.
This commit is contained in:
Navdeep Parhar 2015-03-12 01:30:36 +00:00
parent 5d1e01e943
commit 9d8269b2b3
10 changed files with 663 additions and 1499 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
sys/dev/cxgbe/adapter.h
View file

@ -750,7 +750,7 @@ struct adapter {
void *tom_softc; /* (struct tom_data *) */
struct tom_tunables tt;
void *iwarp_softc; /* (struct c4iw_dev *) */
void *iscsi_softc;
void *iscsi_softc; /* (struct cxgbei_data *) */
#endif
struct l2t_data *l2t; /* L2 table */
struct tid_info tids;

1064
sys/dev/cxgbe/cxgbei/cxgbei.c
View file

File diff suppressed because it is too large Load diff

130
sys/dev/cxgbe/cxgbei/cxgbei.h
View file

@ -28,55 +28,37 @@
#ifndef __CXGBEI_OFLD_H__
#define __CXGBEI_OFLD_H__
#include "mbufq.h"
typedef struct iscsi_socket {
/* iscsi private */
unsigned char s_flag;
unsigned char s_cpuno; /* bind to cpuno */
unsigned char s_mode; /* offload mode */
unsigned char s_txhold;
struct iscsi_socket {
u_char s_dcrc_len;
void *s_conn; /* ic_conn pointer */
struct toepcb *toep;
unsigned char s_ddp_pgidx; /* ddp page selection */
unsigned char s_hcrc_len;
unsigned char s_dcrc_len;
unsigned char filler[1];
/*
* XXXNP: locks on the same line.
* XXXNP: are the locks even needed? Why not use so_snd/so_rcv mtx to
* guard the write and rcv queues?
*/
struct mbufq iscsi_rcvq; /* rx - ULP mbufs */
struct mtx iscsi_rcvq_lock;
unsigned int s_tid; /* for debug only */
unsigned int s_tmax;
unsigned int s_rmax;
unsigned int s_mss;
void *s_odev; /* offload device, if any */
void *s_appdata; /* upperlayer data pointer */
void *s_private; /* underlying socket related info. */
void *s_conn; /* ic_conn pointer */
struct socket *sock;
struct mbuf_head iscsi_rcv_mbufq;/* rx - ULP mbufs */
struct mbuf_head ulp2_writeq; /* tx - ULP mbufs */
struct mbuf_head ulp2_wrq; /* tx wr- ULP mbufs */
struct mbufq ulp2_writeq; /* tx - ULP mbufs */
struct mtx ulp2_writeq_lock;
struct mbufq ulp2_wrq; /* tx wr- ULP mbufs */
struct mtx ulp2_wrq_lock;
struct mbuf *mbuf_ulp_lhdr;
struct mbuf *mbuf_ulp_ldata;
}iscsi_socket;
};
#define ISCSI_SG_SBUF_DMABLE 0x1
#define ISCSI_SG_SBUF_DMA_ONLY 0x2 /*private*/
#define ISCSI_SG_BUF_ALLOC 0x10
#define ISCSI_SG_PAGE_ALLOC 0x20
#define ISCSI_SG_SBUF_MAP_NEEDED 0x40
#define ISCSI_SG_SBUF_MAPPED 0x80
#define ISCSI_SG_SBUF_LISTHEAD 0x100
#define ISCSI_SG_SBUF_LISTTAIL 0x200
#define ISCSI_SG_SBUF_XFER_DONE 0x400
typedef struct cxgbei_sgl {
struct cxgbei_sgl {
int sg_flag;
void *sg_addr;
void *sg_dma_addr;
size_t sg_offset;
size_t sg_length;
} cxgbei_sgl;
};
#define cxgbei_scsi_for_each_sg(_sgl, _sgel, _n, _i) \
for (_i = 0, _sgel = (cxgbei_sgl*) (_sgl); _i < _n; _i++, \
@ -96,14 +78,6 @@ typedef struct cxgbei_sgl {
#define SBUF_ULP_FLAG_PAD_ERROR 0x40
#define SBUF_ULP_FLAG_DATA_DDPED 0x80
/* Flags for return value of CPL message handlers */
enum {
CPL_RET_BUF_DONE = 1, /* buffer processing done buffer may be freed */
CPL_RET_BAD_MSG = 2, /* bad CPL message (e.g., unknown opcode) */
CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */
};
/*
* Similar to tcp_skb_cb but with ULP elements added to support DDP, iSCSI,
* etc.
@ -126,33 +100,55 @@ struct ulp_mbuf_cb {
void *pdu; /* pdu pointer */
};
/* private data for eack scsi task */
typedef struct cxgbei_task_data {
cxgbei_sgl sgl[256];
unsigned int nsge;
unsigned int sc_ddp_tag;
}cxgbei_task_data;
static unsigned char t4tom_cpl_handler_register_flag;
enum {
TOM_CPL_ISCSI_HDR_REGISTERED_BIT,
TOM_CPL_SET_TCB_RPL_REGISTERED_BIT,
TOM_CPL_RX_DATA_DDP_REGISTERED_BIT
/* private data for each scsi task */
struct cxgbei_task_data {
struct cxgbei_sgl sgl[256];
u_int nsge;
u_int sc_ddp_tag;
};
#define ODEV_FLAG_ULP_CRC_ENABLED 0x1
#define ODEV_FLAG_ULP_DDP_ENABLED 0x2
#define ODEV_FLAG_ULP_TX_ALLOC_DIGEST 0x4
#define ODEV_FLAG_ULP_RX_PAD_INCLUDED 0x8
struct cxgbei_ulp2_tag_format {
u_char sw_bits;
u_char rsvd_bits;
u_char rsvd_shift;
u_char filler[1];
uint32_t rsvd_mask;
};
#define ODEV_FLAG_ULP_ENABLED \
(ODEV_FLAG_ULP_CRC_ENABLED | ODEV_FLAG_ULP_DDP_ENABLED)
struct cxgbei_data {
u_int max_txsz;
u_int max_rxsz;
u_int llimit;
u_int ulimit;
u_int nppods;
u_int idx_last;
u_char idx_bits;
uint32_t idx_mask;
uint32_t rsvd_tag_mask;
struct ulp_mbuf_cb * get_ulp_mbuf_cb(struct mbuf *);
int cxgbei_conn_set_ulp_mode(struct socket *, void *);
int cxgbei_conn_close(struct socket *);
struct mtx map_lock;
bus_dma_tag_t ulp_ddp_tag;
unsigned char *colors;
struct cxgbei_ulp2_gather_list **gl_map;
struct cxgbei_ulp2_tag_format tag_format;
};
struct icl_conn;
struct icl_pdu;
struct ulp_mbuf_cb *get_ulp_mbuf_cb(struct mbuf *);
int cxgbei_conn_handoff(struct icl_conn *);
int cxgbei_conn_close(struct icl_conn *);
void cxgbei_conn_task_reserve_itt(void *, void **, void *, unsigned int *);
void cxgbei_conn_transfer_reserve_ttt(void *, void **, void *, unsigned int *);
void cxgbei_cleanup_task(void *, void *);
int cxgbei_conn_xmit_pdu(void *, void *);
int cxgbei_conn_xmit_pdu(struct icl_conn *, struct icl_pdu *);
struct cxgbei_ulp2_pagepod_hdr;
int t4_ddp_set_map(struct cxgbei_data *, void *,
struct cxgbei_ulp2_pagepod_hdr *, u_int, u_int,
struct cxgbei_ulp2_gather_list *, int);
void t4_ddp_clear_map(struct cxgbei_data *, struct cxgbei_ulp2_gather_list *,
u_int, u_int, u_int, struct iscsi_socket *);
#endif

486
sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.c
View file

@ -60,27 +60,6 @@ __FBSDID("$FreeBSD$");
#include "cxgbei.h"
#include "cxgbei_ulp2_ddp.h"
static inline int
cxgbei_counter_dec_and_read(volatile int *p)
{
atomic_subtract_acq_int(p, 1);
return atomic_load_acq_int(p);
}
static inline int
get_order(unsigned long size)
{
int order;
size = (size - 1) >> PAGE_SHIFT;
order = 0;
while (size) {
order++;
size >>= 1;
}
return (order);
}
/*
* Map a single buffer address.
*/
@ -96,25 +75,6 @@ ulp2_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error)
*ba = segs->ds_addr;
}
static int
ulp2_dma_tag_create(struct cxgbei_ulp2_ddp_info *ddp)
{
int rc;
rc = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR,
BUS_SPACE_MAXADDR, NULL, NULL, UINT32_MAX , 8,
BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL, NULL,
&ddp->ulp_ddp_tag);
if (rc != 0) {
printf("%s(%d): bus_dma_tag_create() "
"failed (rc = %d)!\n",
__FILE__, __LINE__, rc);
return rc;
}
return 0;
}
/*
* iSCSI Direct Data Placement
*
@ -129,157 +89,95 @@ ulp2_dma_tag_create(struct cxgbei_ulp2_ddp_info *ddp)
* is the base for ITT/TTT.
*/
unsigned char ddp_page_order[DDP_PGIDX_MAX] = {0, 1, 2, 4};
unsigned char ddp_page_shift[DDP_PGIDX_MAX] = {12, 13, 14, 16};
unsigned char page_idx = DDP_PGIDX_MAX;
static inline int
ddp_find_unused_entries(struct cxgbei_ulp2_ddp_info *ddp,
unsigned int start, unsigned int max,
unsigned int count, unsigned int *idx,
struct cxgbei_ulp2_gather_list *gl)
ddp_find_unused_entries(struct cxgbei_data *ci, u_int start, u_int max,
u_int count, u_int *idx, struct cxgbei_ulp2_gather_list *gl)
{
unsigned int i, j, k;
/* not enough entries */
if ((max - start) < count)
return EBUSY;
if (max - start < count)
return (EBUSY);
max -= count;
mtx_lock(&ddp->map_lock);
mtx_lock(&ci->map_lock);
for (i = start; i < max;) {
for (j = 0, k = i; j < count; j++, k++) {
if (ddp->gl_map[k])
if (ci->gl_map[k])
break;
}
if (j == count) {
for (j = 0, k = i; j < count; j++, k++)
ddp->gl_map[k] = gl;
mtx_unlock(&ddp->map_lock);
ci->gl_map[k] = gl;
mtx_unlock(&ci->map_lock);
*idx = i;
return 0;
return (0);
}
i += j + 1;
}
mtx_unlock(&ddp->map_lock);
return EBUSY;
mtx_unlock(&ci->map_lock);
return (EBUSY);
}
static inline void
ddp_unmark_entries(struct cxgbei_ulp2_ddp_info *ddp,
int start, int count)
ddp_unmark_entries(struct cxgbei_data *ci, u_int start, u_int count)
{
mtx_lock(&ddp->map_lock);
memset(&ddp->gl_map[start], 0,
mtx_lock(&ci->map_lock);
memset(&ci->gl_map[start], 0,
count * sizeof(struct cxgbei_ulp2_gather_list *));
mtx_unlock(&ddp->map_lock);
mtx_unlock(&ci->map_lock);
}
/**
* cxgbei_ulp2_ddp_find_page_index - return ddp page index for a given page size
* @pgsz: page size
* return the ddp page index, if no match is found return DDP_PGIDX_MAX.
*/
int
cxgbei_ulp2_ddp_find_page_index(unsigned long pgsz)
{
int i;
for (i = 0; i < DDP_PGIDX_MAX; i++) {
if (pgsz == (1UL << ddp_page_shift[i]))
return i;
}
CTR1(KTR_CXGBE, "ddp page size 0x%lx not supported.\n", pgsz);
return DDP_PGIDX_MAX;
}
static int
cxgbei_ulp2_ddp_adjust_page_table(void)
{
int i;
unsigned int base_order, order;
if (PAGE_SIZE < (1UL << ddp_page_shift[0])) {
CTR2(KTR_CXGBE, "PAGE_SIZE %u too small, min. %lu.\n",
PAGE_SIZE, 1UL << ddp_page_shift[0]);
return EINVAL;
}
base_order = get_order(1UL << ddp_page_shift[0]);
order = get_order(1 << PAGE_SHIFT);
for (i = 0; i < DDP_PGIDX_MAX; i++) {
/* first is the kernel page size, then just doubling the size */
ddp_page_order[i] = order - base_order + i;
ddp_page_shift[i] = PAGE_SHIFT + i;
}
return 0;
}
static inline void
ddp_gl_unmap(struct toedev *tdev,
struct cxgbei_ulp2_gather_list *gl)
ddp_gl_unmap(struct cxgbei_data *ci, struct cxgbei_ulp2_gather_list *gl)
{
int i;
struct adapter *sc = tdev->tod_softc;
struct cxgbei_ulp2_ddp_info *ddp = sc->iscsi_softc;
if (!gl->pages[0])
return;
for (i = 0; i < gl->nelem; i++) {
bus_dmamap_unload(ddp->ulp_ddp_tag, gl->dma_sg[i].bus_map);
bus_dmamap_destroy(ddp->ulp_ddp_tag, gl->dma_sg[i].bus_map);
bus_dmamap_unload(ci->ulp_ddp_tag, gl->dma_sg[i].bus_map);
bus_dmamap_destroy(ci->ulp_ddp_tag, gl->dma_sg[i].bus_map);
}
}
static inline int
ddp_gl_map(struct toedev *tdev,
struct cxgbei_ulp2_gather_list *gl)
ddp_gl_map(struct cxgbei_data *ci, struct cxgbei_ulp2_gather_list *gl)
{
int i, rc;
bus_addr_t pa;
struct cxgbei_ulp2_ddp_info *ddp;
struct adapter *sc = tdev->tod_softc;
ddp = (struct cxgbei_ulp2_ddp_info *)sc->iscsi_softc;
if (ddp == NULL) {
printf("%s: DDP is NULL tdev:%p sc:%p ddp:%p\n",
__func__, tdev, sc, ddp);
return ENOMEM;
}
mtx_lock(&ddp->map_lock);
MPASS(ci != NULL);
mtx_lock(&ci->map_lock);
for (i = 0; i < gl->nelem; i++) {
rc = bus_dmamap_create(ddp->ulp_ddp_tag, 0,
&gl->dma_sg[i].bus_map);
if (rc != 0) {
printf("%s: unable to map page 0x%p.\n",
__func__, gl->pages[i]);
rc = bus_dmamap_create(ci->ulp_ddp_tag, 0,
&gl->dma_sg[i].bus_map);
if (rc != 0)
goto unmap;
}
rc = bus_dmamap_load(ddp->ulp_ddp_tag, gl->dma_sg[i].bus_map,
rc = bus_dmamap_load(ci->ulp_ddp_tag, gl->dma_sg[i].bus_map,
gl->pages[i], PAGE_SIZE, ulp2_dma_map_addr,
&pa, BUS_DMA_NOWAIT);
if (rc != 0) {
printf("%s:unable to load page 0x%p.\n",
__func__, gl->pages[i]);
if (rc != 0)
goto unmap;
}
gl->dma_sg[i].phys_addr = pa;
}
mtx_unlock(&ddp->map_lock);
mtx_unlock(&ci->map_lock);
return 0;
return (0);
unmap:
if (i) {
unsigned int nelem = gl->nelem;
u_int nelem = gl->nelem;
gl->nelem = i;
ddp_gl_unmap(tdev, gl);
ddp_gl_unmap(ci, gl);
gl->nelem = nelem;
}
return ENOMEM;
return (ENOMEM);
}
/**
@ -297,13 +195,11 @@ unmap:
* memory can be used for ddp. Return NULL otherwise.
*/
struct cxgbei_ulp2_gather_list *
cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec
(unsigned int xferlen, cxgbei_sgl *sgl,
unsigned int sgcnt, void *tdev,
int gfp)
cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(u_int xferlen, struct cxgbei_sgl *sgl,
u_int sgcnt, struct cxgbei_data *ci, int gfp)
{
struct cxgbei_ulp2_gather_list *gl;
cxgbei_sgl *sg = sgl;
struct cxgbei_sgl *sg = sgl;
void *sgpage = (void *)((u64)sg->sg_addr & (~PAGE_MASK));
unsigned int sglen = sg->sg_length;
unsigned int sgoffset = (u64)sg->sg_addr & PAGE_MASK;
@ -320,10 +216,8 @@ cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec
gl = malloc(sizeof(struct cxgbei_ulp2_gather_list) +
npages * (sizeof(struct dma_segments) + sizeof(void *)),
M_DEVBUF, M_NOWAIT | M_ZERO);
if (gl == NULL) {
printf("%s: gl alloc failed\n", __func__);
return NULL;
}
if (gl == NULL)
return (NULL);
gl->pages = (void **)&gl->dma_sg[npages];
gl->length = xferlen;
@ -361,7 +255,7 @@ cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec
}
gl->nelem = ++j;
if (ddp_gl_map(tdev, gl) < 0)
if (ddp_gl_map(ci, gl) < 0)
goto error_out;
return gl;
@ -378,15 +272,17 @@ error_out:
* free a ddp page buffer list resulted from cxgbei_ulp2_ddp_make_gl().
*/
void
cxgbei_ulp2_ddp_release_gl(struct cxgbei_ulp2_gather_list *gl, void *tdev)
cxgbei_ulp2_ddp_release_gl(struct cxgbei_data *ci,
struct cxgbei_ulp2_gather_list *gl)
{
ddp_gl_unmap(tdev, gl);
ddp_gl_unmap(ci, gl);
free(gl, M_DEVBUF);
}
/**
* cxgbei_ulp2_ddp_tag_reserve - set up ddp for a data transfer
* @ddp: adapter's ddp info
* @ci: adapter's ddp info
* @tid: connection id
* @tformat: tag format
* @tagp: contains s/w tag initially, will be updated with ddp/hw tag
@ -397,93 +293,84 @@ cxgbei_ulp2_ddp_release_gl(struct cxgbei_ulp2_gather_list *gl, void *tdev)
* return 0 if success, < 0 otherwise.
*/
int
cxgbei_ulp2_ddp_tag_reserve(struct cxgbei_ulp2_ddp_info *ddp,
void *isock, unsigned int tid,
struct cxgbei_ulp2_tag_format *tformat,
u32 *tagp, struct cxgbei_ulp2_gather_list *gl,
int gfp, int reply)
cxgbei_ulp2_ddp_tag_reserve(struct cxgbei_data *ci, void *isock, u_int tid,
struct cxgbei_ulp2_tag_format *tformat, u32 *tagp,
struct cxgbei_ulp2_gather_list *gl, int gfp, int reply)
{
struct cxgbei_ulp2_pagepod_hdr hdr;
unsigned int npods, idx;
int rv;
u_int npods, idx;
int rc;
u32 sw_tag = *tagp;
u32 tag;
if (page_idx >= DDP_PGIDX_MAX || !ddp || !gl || !gl->nelem ||
gl->length < DDP_THRESHOLD) {
CTR3(KTR_CXGBE, "pgidx %u, xfer %u/%u, NO ddp.\n",
page_idx, gl->length, DDP_THRESHOLD);
return EINVAL;
}
MPASS(ci != NULL);
if (!gl || !gl->nelem || gl->length < DDP_THRESHOLD)
return (EINVAL);
npods = (gl->nelem + IPPOD_PAGES_MAX - 1) >> IPPOD_PAGES_SHIFT;
if (ddp->idx_last == ddp->nppods)
rv = ddp_find_unused_entries(ddp, 0, ddp->nppods,
npods, &idx, gl);
if (ci->idx_last == ci->nppods)
rc = ddp_find_unused_entries(ci, 0, ci->nppods, npods, &idx,
gl);
else {
rv = ddp_find_unused_entries(ddp, ddp->idx_last + 1,
ddp->nppods, npods, &idx, gl);
if (rv && ddp->idx_last >= npods) {
rv = ddp_find_unused_entries(ddp, 0,
min(ddp->idx_last + npods, ddp->nppods),
rc = ddp_find_unused_entries(ci, ci->idx_last + 1,
ci->nppods, npods, &idx, gl);
if (rc && ci->idx_last >= npods) {
rc = ddp_find_unused_entries(ci, 0,
min(ci->idx_last + npods, ci->nppods),
npods, &idx, gl);
}
}
if (rv) {
if (rc) {
CTR3(KTR_CXGBE, "xferlen %u, gl %u, npods %u NO DDP.\n",
gl->length, gl->nelem, npods);
return rv;
return (rc);
}
tag = cxgbei_ulp2_ddp_tag_base(idx, ddp, tformat, sw_tag);
tag = cxgbei_ulp2_ddp_tag_base(idx, ci->colors, tformat, sw_tag);
CTR4(KTR_CXGBE, "%s: sw_tag:0x%x idx:0x%x tag:0x%x\n",
__func__, sw_tag, idx, tag);
hdr.rsvd = 0;
hdr.vld_tid = htonl(F_IPPOD_VALID | V_IPPOD_TID(tid));
hdr.pgsz_tag_clr = htonl(tag & ddp->rsvd_tag_mask);
hdr.pgsz_tag_clr = htonl(tag & ci->rsvd_tag_mask);
hdr.maxoffset = htonl(gl->length);
hdr.pgoffset = htonl(gl->offset);
rv = ddp->ddp_set_map(ddp, isock, &hdr, idx, npods, gl, reply);
if (rv < 0)
rc = t4_ddp_set_map(ci, isock, &hdr, idx, npods, gl, reply);
if (rc < 0)
goto unmark_entries;
ddp->idx_last = idx;
ci->idx_last = idx;
*tagp = tag;
return 0;
return (0);
unmark_entries:
ddp_unmark_entries(ddp, idx, npods);
return rv;
ddp_unmark_entries(ci, idx, npods);
return (rc);
}
/**
* cxgbei_ulp2_ddp_tag_release - release a ddp tag
* @ddp: adapter's ddp info
* @ci: adapter's ddp info
* @tag: ddp tag
* ddp cleanup for a given ddp tag and release all the resources held
*/
void
cxgbei_ulp2_ddp_tag_release(struct cxgbei_ulp2_ddp_info *ddp, u32 tag,
iscsi_socket *isock)
cxgbei_ulp2_ddp_tag_release(struct cxgbei_data *ci, uint32_t tag,
struct iscsi_socket *isock)
{
u32 idx;
uint32_t idx;
if (ddp == NULL) {
CTR2(KTR_CXGBE, "%s:release ddp tag 0x%x, ddp NULL.\n",
__func__, tag);
return;
}
if (isock == NULL)
return;
MPASS(ci != NULL);
MPASS(isock != NULL);
idx = (tag >> IPPOD_IDX_SHIFT) & ddp->idx_mask;
idx = (tag >> IPPOD_IDX_SHIFT) & ci->idx_mask;
CTR3(KTR_CXGBE, "tag:0x%x idx:0x%x nppods:0x%x\n",
tag, idx, ddp->nppods);
if (idx < ddp->nppods) {
struct cxgbei_ulp2_gather_list *gl = ddp->gl_map[idx];
tag, idx, ci->nppods);
if (idx < ci->nppods) {
struct cxgbei_ulp2_gather_list *gl = ci->gl_map[idx];
unsigned int npods;
if (!gl || !gl->nelem) {
@ -495,209 +382,32 @@ cxgbei_ulp2_ddp_tag_release(struct cxgbei_ulp2_ddp_info *ddp, u32 tag,
npods = (gl->nelem + IPPOD_PAGES_MAX - 1) >> IPPOD_PAGES_SHIFT;
CTR3(KTR_CXGBE, "ddp tag 0x%x, release idx 0x%x, npods %u.\n",
tag, idx, npods);
ddp->ddp_clear_map(ddp, gl, tag, idx, npods, isock);
ddp_unmark_entries(ddp, idx, npods);
cxgbei_ulp2_ddp_release_gl(gl, ddp->tdev);
t4_ddp_clear_map(ci, gl, tag, idx, npods, isock);
ddp_unmark_entries(ci, idx, npods);
cxgbei_ulp2_ddp_release_gl(ci, gl);
} else
CTR3(KTR_CXGBE, "ddp tag 0x%x, idx 0x%x > max 0x%x.\n",
tag, idx, ddp->nppods);
tag, idx, ci->nppods);
}
/**
* cxgbei_ulp2_adapter_ddp_info - read the adapter's ddp information
* @ddp: adapter's ddp info
* @tformat: tag format
* @txsz: max tx pdu payload size, filled in by this func.
* @rxsz: max rx pdu payload size, filled in by this func.
* setup the tag format for a given iscsi entity
*/
int
cxgbei_ulp2_adapter_ddp_info(struct cxgbei_ulp2_ddp_info *ddp,
struct cxgbei_ulp2_tag_format *tformat,
unsigned int *txsz, unsigned int *rxsz)
{
unsigned char idx_bits;
if (tformat == NULL)
return EINVAL;
if (ddp == NULL)
return EINVAL;
idx_bits = 32 - tformat->sw_bits;
tformat->sw_bits = ddp->idx_bits;
tformat->rsvd_bits = ddp->idx_bits;
tformat->rsvd_shift = IPPOD_IDX_SHIFT;
tformat->rsvd_mask = (1 << tformat->rsvd_bits) - 1;
CTR4(KTR_CXGBE, "tag format: sw %u, rsvd %u,%u, mask 0x%x.\n",
tformat->sw_bits, tformat->rsvd_bits,
tformat->rsvd_shift, tformat->rsvd_mask);
*txsz = min(ULP2_MAX_PDU_PAYLOAD,
ddp->max_txsz - ISCSI_PDU_NONPAYLOAD_LEN);
*rxsz = min(ULP2_MAX_PDU_PAYLOAD,
ddp->max_rxsz - ISCSI_PDU_NONPAYLOAD_LEN);
CTR4(KTR_CXGBE, "max payload size: %u/%u, %u/%u.\n",
*txsz, ddp->max_txsz, *rxsz, ddp->max_rxsz);
return 0;
}
/**
* cxgbei_ulp2_ddp_cleanup - release the cxgbX adapter's ddp resource
* @tdev: t4cdev adapter
* release all the resource held by the ddp pagepod manager for a given
* adapter if needed
* cxgbei_ddp_cleanup - release the adapter's ddp resources
*/
void
cxgbei_ulp2_ddp_cleanup(struct cxgbei_ulp2_ddp_info **ddp_pp)
cxgbei_ddp_cleanup(struct cxgbei_data *ci)
{
int i = 0;
struct cxgbei_ulp2_ddp_info *ddp = *ddp_pp;
if (ddp == NULL)
return;
CTR2(KTR_CXGBE, "tdev, release ddp 0x%p, ref %d.\n",
ddp, atomic_load_acq_int(&ddp->refcnt));
if (ddp && (cxgbei_counter_dec_and_read(&ddp->refcnt) == 0)) {
*ddp_pp = NULL;
while (i < ddp->nppods) {
struct cxgbei_ulp2_gather_list *gl = ddp->gl_map[i];
if (gl) {
int npods = (gl->nelem + IPPOD_PAGES_MAX - 1)
>> IPPOD_PAGES_SHIFT;
CTR2(KTR_CXGBE,
"tdev, ddp %d + %d.\n", i, npods);
free(gl, M_DEVBUF);
i += npods;
} else
i++;
}
bus_dmamap_unload(ddp->ulp_ddp_tag, ddp->ulp_ddp_map);
cxgbei_ulp2_free_big_mem(ddp);
while (i < ci->nppods) {
struct cxgbei_ulp2_gather_list *gl = ci->gl_map[i];
if (gl) {
int npods = (gl->nelem + IPPOD_PAGES_MAX - 1)
>> IPPOD_PAGES_SHIFT;
free(gl, M_DEVBUF);
i += npods;
} else
i++;
}
}
/**
* ddp_init - initialize the cxgb3/4 adapter's ddp resource
* @tdev_name: device name
* @tdev: device
* @ddp: adapter's ddp info
* @uinfo: adapter's iscsi info
* initialize the ddp pagepod manager for a given adapter
*/
static void
ddp_init(void *tdev,
struct cxgbei_ulp2_ddp_info **ddp_pp,
struct ulp_iscsi_info *uinfo)
{
struct cxgbei_ulp2_ddp_info *ddp = *ddp_pp;
unsigned int ppmax, bits;
int i, rc;
if (uinfo->ulimit <= uinfo->llimit) {
printf("%s: tdev, ddp 0x%x >= 0x%x.\n",
__func__, uinfo->llimit, uinfo->ulimit);
return;
}
if (ddp) {
atomic_add_acq_int(&ddp->refcnt, 1);
CTR2(KTR_CXGBE, "tdev, ddp 0x%p already set up, %d.\n",
ddp, atomic_load_acq_int(&ddp->refcnt));
return;
}
ppmax = (uinfo->ulimit - uinfo->llimit + 1) >> IPPOD_SIZE_SHIFT;
if (ppmax <= 1024) {
CTR3(KTR_CXGBE, "tdev, ddp 0x%x ~ 0x%x, nppod %u < 1K.\n",
uinfo->llimit, uinfo->ulimit, ppmax);
return;
}
bits = (fls(ppmax) - 1) + 1;
if (bits > IPPOD_IDX_MAX_SIZE)
bits = IPPOD_IDX_MAX_SIZE;
ppmax = (1 << (bits - 1)) - 1;
ddp = cxgbei_ulp2_alloc_big_mem(sizeof(struct cxgbei_ulp2_ddp_info) +
ppmax * (sizeof(struct cxgbei_ulp2_gather_list *) +
sizeof(unsigned char)));
if (ddp == NULL) {
CTR1(KTR_CXGBE, "unable to alloc ddp 0x%d, ddp disabled.\n",
ppmax);
return;
}
ddp->colors = (unsigned char *)(ddp + 1);
ddp->gl_map = (struct cxgbei_ulp2_gather_list **)(ddp->colors +
ppmax * sizeof(unsigned char));
*ddp_pp = ddp;
mtx_init(&ddp->map_lock, "ddp lock", NULL,
MTX_DEF | MTX_DUPOK| MTX_RECURSE);
atomic_set_acq_int(&ddp->refcnt, 1);
/* dma_tag create */
rc = ulp2_dma_tag_create(ddp);
if (rc) {
printf("%s: unable to alloc ddp 0x%d, ddp disabled.\n",
__func__, ppmax);
return;
}
ddp->tdev = tdev;
ddp->max_txsz = min(uinfo->max_txsz, ULP2_MAX_PKT_SIZE);
ddp->max_rxsz = min(uinfo->max_rxsz, ULP2_MAX_PKT_SIZE);
ddp->llimit = uinfo->llimit;
ddp->ulimit = uinfo->ulimit;
ddp->nppods = ppmax;
ddp->idx_last = ppmax;
ddp->idx_bits = bits;
ddp->idx_mask = (1 << bits) - 1;
ddp->rsvd_tag_mask = (1 << (bits + IPPOD_IDX_SHIFT)) - 1;
CTR2(KTR_CXGBE,
"gl map 0x%p, idx_last %u.\n", ddp->gl_map, ddp->idx_last);
uinfo->tagmask = ddp->idx_mask << IPPOD_IDX_SHIFT;
for (i = 0; i < DDP_PGIDX_MAX; i++)
uinfo->pgsz_factor[i] = ddp_page_order[i];
uinfo->ulimit = uinfo->llimit + (ppmax << IPPOD_SIZE_SHIFT);
printf("nppods %u, bits %u, mask 0x%x,0x%x pkt %u/%u,"
" %u/%u.\n",
ppmax, ddp->idx_bits, ddp->idx_mask,
ddp->rsvd_tag_mask, ddp->max_txsz, uinfo->max_txsz,
ddp->max_rxsz, uinfo->max_rxsz);
rc = bus_dmamap_create(ddp->ulp_ddp_tag, 0, &ddp->ulp_ddp_map);
if (rc != 0) {
printf("%s: bus_dmamap_Create failed\n", __func__);
return;
}
}
/**
* cxgbei_ulp2_ddp_init - initialize ddp functions
*/
void
cxgbei_ulp2_ddp_init(void *tdev,
struct cxgbei_ulp2_ddp_info **ddp_pp,
struct ulp_iscsi_info *uinfo)
{
if (page_idx == DDP_PGIDX_MAX) {
page_idx = cxgbei_ulp2_ddp_find_page_index(PAGE_SIZE);
if (page_idx == DDP_PGIDX_MAX) {
if (cxgbei_ulp2_ddp_adjust_page_table()) {
CTR1(KTR_CXGBE, "PAGE_SIZE %x, ddp disabled.\n",
PAGE_SIZE);
return;
}
}
page_idx = cxgbei_ulp2_ddp_find_page_index(PAGE_SIZE);
}
ddp_init(tdev, ddp_pp, uinfo);
free(ci->colors, M_CXGBE);
free(ci->gl_map, M_CXGBE);
}

182
sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.h
View file

@ -30,43 +30,6 @@
#ifndef __CXGBEI_ULP2_DDP_H__
#define __CXGBEI_ULP2_DDP_H__
#include <sys/malloc.h>
#include <sys/sglist.h>
#include <sys/pciio.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
#include <vm/uma.h>
/*
* Structure used to return information to the iscsi layer.
*/
struct ulp_iscsi_info {
unsigned int offset;
unsigned int llimit;
unsigned int ulimit;
unsigned int tagmask;
unsigned char pgsz_factor[4];
unsigned int max_rxsz;
unsigned int max_txsz;
};
/*
* struct cxgbei_ulp2_tag_format - cxgbei ulp tag format for an iscsi entity
*
* @sw_bits: # of bits used by iscsi software layer
* @rsvd_bits: # of bits used by h/w
* @rsvd_shift: h/w bits shift left
* @rsvd_mask: reserved bit mask
*/
typedef struct cxgbei_ulp2_tag_format {
unsigned char sw_bits;
unsigned char rsvd_bits;
unsigned char rsvd_shift;
unsigned char filler[1];
uint32_t rsvd_mask;
}cxgbei_ulp2_tag_format;
#define CXGBEI_PAGE_MASK (~(PAGE_SIZE-1))
#define DDP_THRESHOLD 2048
@ -90,7 +53,8 @@ typedef struct cxgbei_ulp2_tag_format {
static inline int
cxgbei_ulp2_is_ddp_tag(struct cxgbei_ulp2_tag_format *tformat, uint32_t tag)
{
return !(tag & (1 << (tformat->rsvd_bits + tformat->rsvd_shift - 1)));
return (!(tag & (1 << (tformat->rsvd_bits + tformat->rsvd_shift - 1))));
}
/*
@ -102,9 +66,10 @@ cxgbei_ulp2_is_ddp_tag(struct cxgbei_ulp2_tag_format *tformat, uint32_t tag)
*/
static inline int
cxgbei_ulp2_sw_tag_usable(struct cxgbei_ulp2_tag_format *tformat,
uint32_t sw_tag)
uint32_t sw_tag)
{
return 1;
return (1); /* XXXNP: huh? */
sw_tag >>= (32 - tformat->rsvd_bits + tformat->rsvd_shift);
return !sw_tag;
@ -160,59 +125,6 @@ struct cxgbei_ulp2_gather_list {
struct dma_segments dma_sg[0];
};
struct cxgbei_ulp2_pagepod_hdr;
/*
* struct cxgbei_ulp2_ddp_info - direct data placement for pdu payload
*
* @list: list head to link elements
* @refcnt: count of iscsi entities using it
* @tdev: pointer to tXcdev used by cxgbX driver
* @max_txsz: max tx packet size for ddp
* @max_rxsz: max rx packet size for ddp
* @llimit: lower bound of the page pod memory
* @ulimit: upper bound of the page pod memory
* @nppods: # of page pod entries
* @idx_last: page pod entry last used
* @idx_bits: # of bits the pagepod index would take
* @idx_mask: pagepod index mask
* @rsvd_tag_mask: tag mask
* @map_lock: lock to synchonize access to the page pod map
* @gl_map: ddp memory gather list
*/
struct cxgbei_ulp2_ddp_info {
SLIST_ENTRY(cxgbei_ulp2_ddp_info) cxgbei_ulp2_ddp_list;
volatile int refcnt;
void *tdev; /* t5odev */
unsigned int max_txsz;
unsigned int max_rxsz;
unsigned int llimit;
unsigned int ulimit;
unsigned int nppods;
unsigned int idx_last;
unsigned char idx_bits;
unsigned char filler[3];
uint32_t idx_mask;
uint32_t rsvd_tag_mask;
bus_addr_t rsvd_page_phys_addr;
int (*ddp_set_map)(struct cxgbei_ulp2_ddp_info *ddp,
void *isock,
struct cxgbei_ulp2_pagepod_hdr *hdr,
unsigned int idx, unsigned int npods,
struct cxgbei_ulp2_gather_list *gl, int reply);
void (*ddp_clear_map)(struct cxgbei_ulp2_ddp_info *ddp,
struct cxgbei_ulp2_gather_list *gl,
unsigned int tag, unsigned int idx,
unsigned int npods,
iscsi_socket *isock);
struct mtx map_lock;
bus_dma_tag_t ulp_ddp_tag;
bus_dmamap_t ulp_ddp_map;
unsigned char *colors;
struct cxgbei_ulp2_gather_list **gl_map;
};
#define IPPOD_SIZE sizeof(struct cxgbei_ulp2_pagepod) /* 64 */
#define IPPOD_SIZE_SHIFT 6
@ -244,16 +156,15 @@ struct cxgbei_ulp2_ddp_info {
#define V_IPPOD_PGSZ(x) ((x) << S_IPPOD_PGSZ)
static inline uint32_t
cxgbei_ulp2_ddp_tag_base(unsigned int idx, struct cxgbei_ulp2_ddp_info *ddp,
struct cxgbei_ulp2_tag_format *tformat, uint32_t sw_tag)
cxgbei_ulp2_ddp_tag_base(u_int idx, u_char *colors,
struct cxgbei_ulp2_tag_format *tformat, uint32_t sw_tag)
{
ddp->colors[idx]++;
if (ddp->colors[idx] == (1 << IPPOD_IDX_SHIFT))
ddp->colors[idx] = 0;
if (__predict_false(++colors[idx] == 1 << IPPOD_IDX_SHIFT))
colors[idx] = 0;
sw_tag <<= (tformat->rsvd_bits + tformat->rsvd_shift);
sw_tag <<= tformat->rsvd_bits + tformat->rsvd_shift;
return sw_tag | (idx << 6) | ddp->colors[idx];
return (sw_tag | idx << IPPOD_IDX_SHIFT | colors[idx]);
}
#define ISCSI_PDU_NONPAYLOAD_LEN 312 /* bhs(48) + ahs(256) + digest(8) */
@ -284,65 +195,20 @@ struct cxgbei_ulp2_pagepod {
uint64_t addr[IPPOD_PAGES_MAX + 1];
};
/*
* ddp page size array
*/
#define DDP_PGIDX_MAX 4
extern unsigned char ddp_page_order[DDP_PGIDX_MAX];
extern unsigned char page_idx;
int cxgbei_ulp2_ddp_tag_reserve(struct cxgbei_data *, void *, unsigned int,
struct cxgbei_ulp2_tag_format *, uint32_t *,
struct cxgbei_ulp2_gather_list *, int , int );
void cxgbei_ulp2_ddp_tag_release(struct cxgbei_data *, uint32_t,
struct iscsi_socket *);
struct cxgbei_ulp2_gather_list *cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(u_int,
struct cxgbei_sgl *, u_int, struct cxgbei_data *, int);
void cxgbei_ulp2_ddp_release_gl(struct cxgbei_data *,
struct cxgbei_ulp2_gather_list *);
/*
* large memory chunk allocation/release
* use vmalloc() if kmalloc() fails
*/
static inline void *
cxgbei_ulp2_alloc_big_mem(unsigned int size)
{
void *p = NULL;
int cxgbei_ulp2_ddp_find_page_index(u_long);
int cxgbei_ulp2_adapter_ddp_info(struct cxgbei_data *,
struct cxgbei_ulp2_tag_format *);
p = malloc(size, M_TEMP, M_NOWAIT | M_ZERO);
return p;
}
static inline void
cxgbei_ulp2_free_big_mem(void *addr)
{
free(addr, M_TEMP);
}
int cxgbei_ulp2_ddp_tag_reserve(struct cxgbei_ulp2_ddp_info *,
void *, unsigned int ,
struct cxgbei_ulp2_tag_format *, uint32_t *,
struct cxgbei_ulp2_gather_list *, int , int );
void cxgbei_ulp2_ddp_tag_release(struct cxgbei_ulp2_ddp_info *,
uint32_t, iscsi_socket *);
struct cxgbei_ulp2_gather_list *cxgbei_ulp2_ddp_make_gl(unsigned int ,
struct sglist *,
unsigned int ,
struct pci_conf *,
int);
struct cxgbei_ulp2_gather_list *cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(
unsigned int,
cxgbei_sgl *,
unsigned int,
void *,
int);
void cxgbei_ulp2_ddp_release_gl(struct cxgbei_ulp2_gather_list *, void *);
int cxgbei_ulp2_ddp_find_page_index(unsigned long);
int cxgbei_ulp2_adapter_ddp_info(struct cxgbei_ulp2_ddp_info *,
struct cxgbei_ulp2_tag_format *,
unsigned int *, unsigned int *);
void cxgbei_ulp2_ddp_cleanup(struct cxgbei_ulp2_ddp_info **);
void cxgbei_ulp2_ddp_init(void *,
struct cxgbei_ulp2_ddp_info **,
struct ulp_iscsi_info *);
int cxgbei_ulp2_init(void);
void cxgbei_ulp2_exit(void);
void cxgbei_ddp_cleanup(struct cxgbei_data *);
#endif

10
sys/dev/cxgbe/cxgbei/icl_cxgbei.c
View file

@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$");
#include <sys/systm.h>
#include <sys/sx.h>
#include <sys/uio.h>
#include <machine/bus.h>
#include <vm/uma.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
@ -446,6 +447,7 @@ icl_cxgbei_conn_free(struct icl_conn *ic)
refcount_release(&icl_ncons);
}
/* XXXNP: what is this for? There's no conn_start method. */
static int
icl_conn_start(struct icl_conn *ic)
{
@ -572,9 +574,8 @@ icl_cxgbei_conn_handoff(struct icl_conn *ic, int fd)
ICL_CONN_UNLOCK(ic);
error = icl_conn_start(ic);
if(!error) {
cxgbei_conn_set_ulp_mode(ic->ic_socket, ic);
}
if (!error)
cxgbei_conn_handoff(ic);
return (error);
}
@ -620,7 +621,7 @@ icl_cxgbei_conn_close(struct icl_conn *ic)
//ICL_DEBUG("send/receive threads terminated");
ICL_CONN_UNLOCK(ic);
cxgbei_conn_close(ic->ic_socket);
cxgbei_conn_close(ic);
soclose(ic->ic_socket);
ICL_CONN_LOCK(ic);
ic->ic_socket = NULL;
@ -690,6 +691,7 @@ icl_cxgbei_conn_task_setup(struct icl_conn *ic, struct ccb_scsiio *csio,
void
icl_cxgbei_conn_task_done(struct icl_conn *ic, void *prv)
{
cxgbei_cleanup_task(ic, prv);
uma_zfree(icl_transfer_zone, prv);
}

121
sys/dev/cxgbe/cxgbei/mbufq.h
View file

@ -1,121 +0,0 @@
/**************************************************************************
Copyright (c) 2007-2008, Chelsio Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Neither the name of the Chelsio Corporation nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
$FreeBSD$
***************************************************************************/
#ifndef CXGB_MBUFQ_H_
#define CXGB_MBUFQ_H_
struct mbuf_head {
struct mbuf *head;
struct mbuf *tail;
uint32_t qlen;
uint32_t qsize;
struct mtx lock;
};
static __inline void
mbufq_init(struct mbuf_head *l)
{
l->head = l->tail = NULL;
l->qlen = l->qsize = 0;
}
static __inline int
mbufq_empty(struct mbuf_head *l)
{
return (l->head == NULL);
}
static __inline int
mbufq_len(struct mbuf_head *l)
{
return (l->qlen);
}
static __inline int
mbufq_size(struct mbuf_head *l)
{
return (l->qsize);
}
static __inline int
mbufq_head_size(struct mbuf_head *l)
{
return (l->head ? l->head->m_pkthdr.len : 0);
}
static __inline void
mbufq_tail(struct mbuf_head *l, struct mbuf *m)
{
l->qlen++;
if (l->head == NULL)
l->head = m;
else
l->tail->m_nextpkt = m;
l->tail = m;
l->qsize += m->m_pkthdr.len;
}
static __inline struct mbuf *
mbufq_dequeue(struct mbuf_head *l)
{
struct mbuf *m;
m = l->head;
if (m) {
if (m == l->tail)
l->head = l->tail = NULL;
else
l->head = m->m_nextpkt;
m->m_nextpkt = NULL;
l->qlen--;
l->qsize -= m->m_pkthdr.len;
}
return (m);
}
static __inline struct mbuf *
mbufq_peek(const struct mbuf_head *l)
{
return (l->head);
}
static __inline void
mbufq_append(struct mbuf_head *a, struct mbuf_head *b)
{
if (a->tail)
a->tail->m_nextpkt = b->head;
if (b->tail)
a->tail = b->tail;
a->qlen += b->qlen;
a->qsize += b->qsize;
}
#endif /* CXGB_MBUFQ_H_ */

145
sys/dev/cxgbe/tom/t4_cpl_io.c
View file

@ -805,6 +805,10 @@ t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
close_conn(sc, toep);
}
void (*cxgbei_fw4_ack)(struct toepcb *, int);
struct mbuf *(*cxgbei_writeq_len)(struct toepcb *, int *);
struct mbuf *(*cxgbei_writeq_next)(struct toepcb *);
/* Send ULP data over TOE using TX_DATA_WR. We send whole mbuf at once */
void
t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
@ -840,7 +844,7 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
if (__predict_false(toep->flags & TPF_TX_SUSPENDED))
return;
sndptr = t4_queue_iscsi_callback(so, toep, 1, &qlen);
sndptr = cxgbei_writeq_len(toep, &qlen);
if (!qlen)
return;
@ -850,8 +854,7 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
max_nsegs = max_dsgl_nsegs(tx_credits);
if (drop) {
t4_cpl_iscsi_callback(toep->td, toep, &drop,
CPL_FW4_ACK);
cxgbei_fw4_ack(toep, drop);
drop = 0;
}
@ -951,7 +954,7 @@ t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
tp->snd_max += ulp_len;
/* goto next mbuf */
sndptr = m = t4_queue_iscsi_callback(so, toep, 2, &qlen);
sndptr = m = cxgbei_writeq_next(toep);
toep->flags |= TPF_TX_DATA_SENT;
if (toep->tx_credits < MIN_OFLD_TX_CREDITS) {
@ -1268,91 +1271,6 @@ abort_status_to_errno(struct tcpcb *tp, unsigned int abort_reason)
}
}
int
cpl_not_handled(struct sge_iq *, const struct rss_header *, struct mbuf *);
/*
* tom_cpl_iscsi_callback -
* iscsi and tom would share the following cpl messages, so when any of these
* message is received, after tom is done with processing it, the messages
* needs to be forwarded to iscsi for further processing:
* - CPL_SET_TCB_RPL
* - CPL_RX_DATA_DDP
*/
void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *, void *,
unsigned int);
struct mbuf *(*tom_queue_iscsi_callback)(struct socket *, unsigned int, int *);
/*
* Check if the handler function is set for a given CPL
* return 0 if the function is NULL or cpl_not_handled, 1 otherwise.
*/
int
t4tom_cpl_handler_registered(struct adapter *sc, unsigned int opcode)
{
MPASS(opcode < nitems(sc->cpl_handler));
return (sc->cpl_handler[opcode] &&
sc->cpl_handler[opcode] != cpl_not_handled);
}
/*
* set the tom_cpl_iscsi_callback function, this function should be used
* whenever both toe and iscsi need to process the same cpl msg.
*/
void
t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *, struct socket *,
void *, unsigned int))
{
tom_cpl_iscsi_callback = fp;
}
void
t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *,
unsigned int, int *qlen))
{
tom_queue_iscsi_callback = fp;
}
int
t4_cpl_iscsi_callback(struct tom_data *td, struct toepcb *toep, void *m,
unsigned int opcode)
{
struct socket *so;
if (opcode == CPL_FW4_ACK)
so = toep->inp->inp_socket;
else {
INP_WLOCK(toep->inp);
so = toep->inp->inp_socket;
INP_WUNLOCK(toep->inp);
}
if (tom_cpl_iscsi_callback && so) {
if (toep->ulp_mode == ULP_MODE_ISCSI) {
tom_cpl_iscsi_callback(td, so, m, opcode);
return (0);
}
}
return (1);
}
struct mbuf *
t4_queue_iscsi_callback(struct socket *so, struct toepcb *toep,
unsigned int cmd, int *qlen)
{
if (tom_queue_iscsi_callback && so) {
if (toep->ulp_mode == ULP_MODE_ISCSI)
return (tom_queue_iscsi_callback(so, cmd, qlen));
}
return (NULL);
}
/*
* TCP RST from the peer, timeout, or some other such critical error.
*/
@ -1748,16 +1666,32 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
t4_push_frames(sc, toep, plen);
} else if (plen > 0) {
struct sockbuf *sb = &so->so_snd;
int sbu;
if (toep->ulp_mode == ULP_MODE_ISCSI)
t4_cpl_iscsi_callback(toep->td, toep, &plen,
CPL_FW4_ACK);
else {
SOCKBUF_LOCK(sb);
SOCKBUF_LOCK(sb);
sbu = sbused(sb);
if (toep->ulp_mode == ULP_MODE_ISCSI) {
if (__predict_false(sbu > 0)) {
/*
* The data trasmitted before the tid's ULP mode
* changed to ISCSI is still in so_snd.
* Incoming credits should account for so_snd
* first.
*/
sbdrop_locked(sb, min(sbu, plen));
plen -= min(sbu, plen);
}
/* XXXNP: sowwakeup_locked causes a LOR. */
SOCKBUF_UNLOCK(sb);
if (__predict_true(plen > 0))
cxgbei_fw4_ack(toep, plen);
} else {
sbdrop_locked(sb, plen);
sowwakeup_locked(so);
SOCKBUF_UNLOCK_ASSERT(sb);
sowwakeup_locked(so); /* unlocks so_snd */
}
SOCKBUF_UNLOCK_ASSERT(sb);
}
INP_WUNLOCK(inp);
@ -1781,14 +1715,21 @@ do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
if (is_ftid(sc, tid))
return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */
else {
struct toepcb *toep = lookup_tid(sc, tid);
t4_cpl_iscsi_callback(toep->td, toep, m, CPL_SET_TCB_RPL);
return (0);
}
/*
* TOM and/or other ULPs don't request replies for CPL_SET_TCB or
* CPL_SET_TCB_FIELD requests. This can easily change and when it does
* the dispatch code will go here.
*/
#ifdef INVARIANTS
panic("%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p", __func__,
tid, iq);
#else
log(LOG_ERR, "%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p\n",
__func__, tid, iq);
#endif
CXGBE_UNIMPLEMENTED(__func__);
return (0);
}
void

9
sys/dev/cxgbe/tom/t4_ddp.c
View file

@ -437,6 +437,8 @@ wakeup:
F_DDP_INVALID_TAG | F_DDP_COLOR_ERR | F_DDP_TID_MISMATCH |\
F_DDP_INVALID_PPOD | F_DDP_HDRCRC_ERR | F_DDP_DATACRC_ERR)
void (*cxgbei_rx_data_ddp)(struct toepcb *, const struct cpl_rx_data_ddp *);
static int
do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
@ -445,7 +447,6 @@ do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
unsigned int tid = GET_TID(cpl);
uint32_t vld;
struct toepcb *toep = lookup_tid(sc, tid);
struct tom_data *td = toep->td;
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
@ -457,10 +458,10 @@ do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
panic("%s: DDP error 0x%x (tid %d, toep %p)",
__func__, vld, tid, toep);
}
if (toep->ulp_mode == ULP_MODE_ISCSI) {
if (!t4_cpl_iscsi_callback(td, toep, (void *)cpl,
CPL_RX_DATA_DDP))
return (0);
cxgbei_rx_data_ddp(toep, cpl);
return (0);
}
handle_ddp_data(toep, cpl->u.ddp_report, cpl->seq, be16toh(cpl->len));

13
sys/dev/cxgbe/tom/t4_tom.h
View file

@ -115,6 +115,7 @@ struct toepcb {
int rx_credits; /* rx credits (in bytes) to be returned to hw */
u_int ulp_mode; /* ULP mode */
void *ulpcb;
u_int ddp_flags;
struct ddp_buffer *db[2];
@ -288,17 +289,5 @@ void insert_ddp_data(struct toepcb *, uint32_t);
/* ULP related */
#define CXGBE_ISCSI_MBUF_TAG 50
int t4tom_cpl_handler_registered(struct adapter *, unsigned int);
void t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *,
struct socket *, void *, unsigned int));
void t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *,
unsigned int, int *));
void t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int);
int t4_cpl_iscsi_callback(struct tom_data *, struct toepcb *, void *, uint32_t);
struct mbuf *t4_queue_iscsi_callback(struct socket *, struct toepcb *, uint32_t,
int *);
extern void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *,
void *, unsigned int);
extern struct mbuf *(*tom_queue_iscsi_callback)(struct socket*, unsigned int,
int *);
#endif