Add support for passthrough NVMe commands.

This includes a new IOCTL to support a generic method for nvmecontrol(8) to pass
IDENTIFY, GET_LOG_PAGE, GET_FEATURES and other commands to the controller, rather than
separate IOCTLs for each.

Sponsored by:	Intel

sys/dev/nvme/nvme.h

@@ -38,6 +38,7 @@
 #define	NVME_IO_TEST			_IOWR('n', 2, struct nvme_io_test)
 #define	NVME_BIO_TEST			_IOWR('n', 4, struct nvme_io_test)
 #define	NVME_RESET_CONTROLLER		_IO('n', 5)
+#define	NVME_PASSTHROUGH_CMD		_IOWR('n', 6, struct nvme_pt_command)
 
 /*
  * Use to mark a command to apply to all namespaces, or to retrieve global
@@ -716,6 +717,59 @@ enum nvme_io_test_flags {
 	NVME_TEST_FLAG_REFTHREAD =	0x1,
 };
 
+struct nvme_pt_command {
+
+	/*
+	 * cmd is used to specify a passthrough command to a controller or
+	 *  namespace.
+	 *
+	 * The following fields from cmd may be specified by the caller:
+	 *	* opc  (opcode)
+	 *	* nsid (namespace id) - for admin commands only
+	 *	* cdw10-cdw15
+	 *
+	 * Remaining fields must be set to 0 by the caller.
+	 */
+	struct nvme_command	cmd;
+
+	/*
+	 * cpl returns completion status for the passthrough command
+	 *  specified by cmd.
+	 *
+	 * The following fields will be filled out by the driver, for
+	 *  consumption by the caller:
+	 *	* cdw0
+	 *	* status (except for phase)
+	 *
+	 * Remaining fields will be set to 0 by the driver.
+	 */
+	struct nvme_completion	cpl;
+
+	/* buf is the data buffer associated with this passthrough command. */
+	void *			buf;
+
+	/*
+	 * len is the length of the data buffer associated with this
+	 *  passthrough command.
+	 */
+	uint32_t		len;
+
+	/*
+	 * is_read = 1 if the passthrough command will read data into the
+	 *  supplied buffer.
+	 *
+	 * is_read = 0 if the passthrough command will write data into the
+	 *  supplied buffer.
+	 */
+	uint32_t		is_read;
+
+	/*
+	 * driver_lock is used by the driver only.  It must be set to 0
+	 *  by the caller.
+	 */
+	struct mtx *		driver_lock;
+};
+
 #define nvme_completion_is_error(cpl)					\
 	((cpl)->status.sc != 0 || (cpl)->status.sct != 0)
 
@@ -740,6 +794,11 @@ enum nvme_namespace_flags {
 	NVME_NS_FLUSH_SUPPORTED		= 0x2,
 };
 
+int	nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr,
+				   struct nvme_pt_command *pt,
+				   uint32_t nsid, int is_user_buffer,
+				   int is_admin_cmd);
+
 /* Admin functions */
 void	nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr,
 				   uint8_t feature, uint32_t cdw11,

sys/dev/nvme/nvme_ctrlr.c

@@ -28,10 +28,14 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/buf.h>
 #include <sys/bus.h>
 #include <sys/conf.h>
 #include <sys/ioccom.h>
+#include <sys/proc.h>
 #include <sys/smp.h>
+#include <sys/uio.h>
 
 #include <dev/pci/pcireg.h>
 #include <dev/pci/pcivar.h>
@@ -878,12 +882,103 @@ nvme_ctrlr_configure_intx(struct nvme_controller *ctrlr)
 	return (0);
 }
 
+static void
+nvme_pt_done(void *arg, const struct nvme_completion *cpl)
+{
+	struct nvme_pt_command *pt = arg;
+
+	bzero(&pt->cpl, sizeof(pt->cpl));
+	pt->cpl.cdw0 = cpl->cdw0;
+	pt->cpl.status = cpl->status;
+	pt->cpl.status.p = 0;
+
+	mtx_lock(pt->driver_lock);
+	wakeup(pt);
+	mtx_unlock(pt->driver_lock);
+}
+
+int
+nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr,
+    struct nvme_pt_command *pt, uint32_t nsid, int is_user_buffer,
+    int is_admin_cmd)
+{
+	struct nvme_request	*req;
+	struct mtx		*mtx;
+	struct buf		*buf = NULL;
+	int			ret = 0;
+
+	if (pt->len > 0)
+		if (is_user_buffer) {
+			/*
+			 * Ensure the user buffer is wired for the duration of
+			 *  this passthrough command.
+			 */
+			PHOLD(curproc);
+			buf = getpbuf(NULL);
+			buf->b_saveaddr = buf->b_data;
+			buf->b_data = pt->buf;
+			buf->b_bufsize = pt->len;
+			buf->b_iocmd = pt->is_read ? BIO_READ : BIO_WRITE;
+#ifdef NVME_UNMAPPED_BIO_SUPPORT
+			if (vmapbuf(buf, 1) < 0) {
+#else
+			if (vmapbuf(buf) < 0) {
+#endif
+				ret = EFAULT;
+				goto err;
+			}
+			req = nvme_allocate_request_vaddr(buf->b_data, pt->len, 
+			    nvme_pt_done, pt);
+		} else
+			req = nvme_allocate_request_vaddr(pt->buf, pt->len,
+			    nvme_pt_done, pt);
+	else
+		req = nvme_allocate_request_null(nvme_pt_done, pt);
+
+	req->cmd.opc	= pt->cmd.opc;
+	req->cmd.cdw10	= pt->cmd.cdw10;
+	req->cmd.cdw11	= pt->cmd.cdw11;
+	req->cmd.cdw12	= pt->cmd.cdw12;
+	req->cmd.cdw13	= pt->cmd.cdw13;
+	req->cmd.cdw14	= pt->cmd.cdw14;
+	req->cmd.cdw15	= pt->cmd.cdw15;
+
+	req->cmd.nsid = nsid;
+
+	if (is_admin_cmd)
+		mtx = &ctrlr->lock;
+	else
+		mtx = &ctrlr->ns[nsid-1].lock;
+
+	mtx_lock(mtx);
+	pt->driver_lock = mtx;
+
+	if (is_admin_cmd)
+		nvme_ctrlr_submit_admin_request(ctrlr, req);
+	else
+		nvme_ctrlr_submit_io_request(ctrlr, req);
+
+	mtx_sleep(pt, mtx, PRIBIO, "nvme_pt", 0);
+	mtx_unlock(mtx);
+
+	pt->driver_lock = NULL;
+
+err:
+	if (buf != NULL) {
+		relpbuf(buf, NULL);
+		PRELE(curproc);
+	}
+
+	return (ret);
+}
+
 static int
 nvme_ctrlr_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag,
     struct thread *td)
 {
 	struct nvme_completion_poll_status	status;
 	struct nvme_controller			*ctrlr;
+	struct nvme_pt_command			*pt;
 
 	ctrlr = cdev->si_drv1;
 
@@ -912,6 +1007,10 @@ nvme_ctrlr_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag,
 	case NVME_RESET_CONTROLLER:
 		nvme_ctrlr_reset(ctrlr);
 		break;
+	case NVME_PASSTHROUGH_CMD:
+		pt = (struct nvme_pt_command *)arg;
+		return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, pt->cmd.nsid,
+		    1 /* is_user_buffer */, 1 /* is_admin_cmd */));
 	default:
 		return (ENOTTY);
 	}

sys/dev/nvme/nvme_ns.c

@@ -48,6 +48,7 @@ nvme_ns_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag,
 	struct nvme_completion_poll_status	status;
 	struct nvme_namespace			*ns;
 	struct nvme_controller			*ctrlr;
+	struct nvme_pt_command			*pt;
 
 	ns = cdev->si_drv1;
 	ctrlr = ns->ctrlr;
@@ -78,6 +79,10 @@ nvme_ns_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag,
 	case NVME_BIO_TEST:
 		nvme_ns_test(ns, cmd, arg);
 		break;
+	case NVME_PASSTHROUGH_CMD:
+		pt = (struct nvme_pt_command *)arg;
+		return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, ns->id, 
+		    1 /* is_user_buffer */, 0 /* is_admin_cmd */));
 	case DIOCGMEDIASIZE:
 		*(off_t *)arg = (off_t)nvme_ns_get_size(ns);
 		break;