if_ntb: MFV e26a5843: Move MW/DB management to if_ntb

This is the last e26a5843 patch.  The general thrust of the rewrite was
to move more responsibility for Memory Window and Doorbell interrupt
management from the ntb_hw driver to if_ntb.

A number of APIs have been added, removed, or replaced.  The old
DB callback mechanism has been excised.  Instead, callers (if_ntb) are
responsible for configuring MWs and handling their interrupts more
directly.

This adds a tunable, hw.ntb.max_mw_size, allowing users to limit the
size of memory windows used by if_ntb (identical to the Linux modparam
of the same name).

Despite attempts to keep mechanical name changes to separate commits,
some have snuck in here.  At least the driver should be much more
similar to the latest Linux one now -- making porting fixes easier.

Authored by:	Allen Hubbe
Obtained from:	Linux (Dual BSD/GPL driver)
Sponsored by:	EMC / Isilon Storage Division

sys/dev/ntb/ntb_hw/ntb_hw.h

@@ -34,36 +34,53 @@ struct ntb_softc;
 
 #define NTB_MAX_NUM_MW	3
 
-enum ntb_link_event {
-	NTB_LINK_DOWN = 0,
-	NTB_LINK_UP,
+enum ntb_speed {
+	NTB_SPEED_AUTO = -1,
+	NTB_SPEED_NONE = 0,
+	NTB_SPEED_GEN1 = 1,
+	NTB_SPEED_GEN2 = 2,
+	NTB_SPEED_GEN3 = 3,
 };
 
-enum ntb_hw_event {
-	NTB_EVENT_SW_EVENT0 = 0,
-	NTB_EVENT_SW_EVENT1,
-	NTB_EVENT_SW_EVENT2,
-	NTB_EVENT_HW_ERROR,
-	NTB_EVENT_HW_LINK_UP,
-	NTB_EVENT_HW_LINK_DOWN,
+enum ntb_width {
+	NTB_WIDTH_AUTO = -1,
+	NTB_WIDTH_NONE = 0,
+	NTB_WIDTH_1 = 1,
+	NTB_WIDTH_2 = 2,
+	NTB_WIDTH_4 = 4,
+	NTB_WIDTH_8 = 8,
+	NTB_WIDTH_12 = 12,
+	NTB_WIDTH_16 = 16,
+	NTB_WIDTH_32 = 32,
 };
 
 SYSCTL_DECL(_hw_ntb);
 
-typedef int (*ntb_db_callback)(void *data, int db_num);
-typedef void (*ntb_event_callback)(void *data, enum ntb_hw_event event);
+typedef void (*ntb_db_callback)(void *data, int vector);
+typedef void (*ntb_event_callback)(void *data);
+
+struct ntb_ctx_ops {
+	ntb_event_callback	link_event;
+	ntb_db_callback		db_event;
+};
+
+device_t ntb_get_device(struct ntb_softc *);
+
+bool ntb_link_is_up(struct ntb_softc *, enum ntb_speed *, enum ntb_width *);
+void ntb_link_event(struct ntb_softc *);
+int ntb_link_enable(struct ntb_softc *, enum ntb_speed, enum ntb_width);
+int ntb_link_disable(struct ntb_softc *);
+
+int ntb_set_ctx(struct ntb_softc *, void *, const struct ntb_ctx_ops *);
+void *ntb_get_ctx(struct ntb_softc *, const struct ntb_ctx_ops **);
+void ntb_clear_ctx(struct ntb_softc *);
+
+uint8_t ntb_mw_count(struct ntb_softc *);
+int ntb_mw_get_range(struct ntb_softc *, unsigned mw_idx, vm_paddr_t *base,
+    void **vbase, size_t *size, size_t *align, size_t *align_size);
+int ntb_mw_set_trans(struct ntb_softc *, unsigned mw_idx, bus_addr_t, size_t);
+int ntb_mw_clear_trans(struct ntb_softc *, unsigned mw_idx);
 
-int ntb_register_event_callback(struct ntb_softc *ntb, ntb_event_callback func);
-void ntb_unregister_event_callback(struct ntb_softc *ntb);
-int ntb_register_db_callback(struct ntb_softc *ntb, unsigned int idx,
-    void *data, ntb_db_callback func);
-void ntb_unregister_db_callback(struct ntb_softc *ntb, unsigned int idx);
-void *ntb_find_transport(struct ntb_softc *ntb);
-struct ntb_softc *ntb_register_transport(struct ntb_softc *ntb,
-    void *transport);
-void ntb_unregister_transport(struct ntb_softc *ntb);
-uint8_t ntb_get_max_cbs(struct ntb_softc *ntb);
-uint8_t ntb_mw_count(struct ntb_softc *ntb);
 uint8_t ntb_get_max_spads(struct ntb_softc *ntb);
 int ntb_spad_write(struct ntb_softc *ntb, unsigned int idx, uint32_t val);
 int ntb_spad_read(struct ntb_softc *ntb, unsigned int idx, uint32_t *val);
@@ -71,14 +88,16 @@ int ntb_peer_spad_write(struct ntb_softc *ntb, unsigned int idx,
     uint32_t val);
 int ntb_peer_spad_read(struct ntb_softc *ntb, unsigned int idx,
     uint32_t *val);
-void *ntb_get_mw_vbase(struct ntb_softc *ntb, unsigned int mw);
-bus_addr_t ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw);
-u_long ntb_get_mw_size(struct ntb_softc *ntb, unsigned int mw);
-void ntb_set_mw_addr(struct ntb_softc *ntb, unsigned int mw, uint64_t addr);
-void ntb_peer_db_set(struct ntb_softc *ntb, uint64_t bit);
-bus_addr_t ntb_get_peer_db_addr(struct ntb_softc *ntb, vm_size_t *sz_out);
-bool ntb_link_is_up(struct ntb_softc *ntb);
-device_t ntb_get_device(struct ntb_softc *ntb);
+
+uint64_t ntb_db_valid_mask(struct ntb_softc *);
+bus_addr_t ntb_get_peer_db_addr(struct ntb_softc *, vm_size_t *sz_out);
+
+void ntb_db_clear(struct ntb_softc *, uint64_t bits);
+void ntb_db_clear_mask(struct ntb_softc *, uint64_t bits);
+uint64_t ntb_db_read(struct ntb_softc *);
+void ntb_db_set_mask(struct ntb_softc *, uint64_t bits);
+uint64_t ntb_db_vector_mask(struct ntb_softc *, int vector);
+void ntb_peer_db_set(struct ntb_softc *, uint64_t bits);
 
 /* Hardware owns the low 32 bits of features. */
 #define NTB_BAR_SIZE_4K		(1 << 0)

sys/dev/ntb/ntb_hw/ntb_regs.h

@@ -33,6 +33,7 @@
 #define NTB_LINK_STATUS_ACTIVE	0x2000
 #define NTB_LINK_SPEED_MASK	0x000f
 #define NTB_LINK_WIDTH_MASK	0x03f0
+#define NTB_LNK_STA_WIDTH(sta)	(((sta) & NTB_LINK_WIDTH_MASK) >> 4)
 
 #define XEON_SNB_MW_COUNT	2
 #define XEON_HSX_SPLIT_MW_COUNT	3