From 22bb3201ac59593fa50cc4ee4f8d6cf0ea93268f Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Mon, 12 Jan 2015 07:48:22 +0000
Subject: [PATCH] Fix several issues with /dev/mem and /dev/kmem devices on
 amd64.

For /dev/mem, when requested physical address is not accessible by the
direct map, do temporal remaping with the caching attribute
'uncached'.  Limit the accessible addresses by MAXPHYADDR, since the
architecture disallowes writing non-zero into reserved bits of ptes
(or setting garbage into NX).

For /dev/kmem, only access existing kernel mappings for direct map
region.  For all other addresses, obtain a physical address of the
mapping and fall back to the /dev/mem mechanism.  This ensures that
/dev/kmem i/o does not fault even if the accessed region is changed in
parallel, by using either direct map or temporal mapping.

For both devices, operate on one page by iteration.  Do not return
error if any bytes were moved around, return the (partial) bytes count
to userspace.

Reviewed by:	alc
Tested by:	pho
Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
---
 sys/amd64/amd64/mem.c | 108 +++++++++++++++++++++++-------------------
 1 file changed, 58 insertions(+), 50 deletions(-)

diff --git a/sys/amd64/amd64/mem.c b/sys/amd64/amd64/mem.c
index b6b8a6685e7..3a1f4a4c8a4 100644
--- a/sys/amd64/amd64/mem.c
+++ b/sys/amd64/amd64/mem.c
@@ -58,6 +58,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/uio.h>
 
+#include <machine/md_var.h>
 #include <machine/specialreg.h>
 #include <machine/vmparam.h>
 
@@ -77,13 +78,15 @@ int
 memrw(struct cdev *dev, struct uio *uio, int flags)
 {
 	struct iovec *iov;
-	u_long c, v, vd;
-	int error, o, sflags;
-	vm_offset_t addr, eaddr;
+	void *p;
+	ssize_t orig_resid;
+	u_long v, vd;
+	u_int c;
+	int error, sflags;
 
 	error = 0;
-	c = 0;
 	sflags = curthread_pflags_set(TDP_DEVMEMIO);
+	orig_resid = uio->uio_resid;
 	while (uio->uio_resid > 0 && error == 0) {
 		iov = uio->uio_iov;
 		if (iov->iov_len == 0) {
@@ -93,63 +96,68 @@ memrw(struct cdev *dev, struct uio *uio, int flags)
 				panic("memrw");
 			continue;
 		}
-		if (dev2unit(dev) == CDEV_MINOR_MEM) {
-			v = uio->uio_offset;
-kmemphys:
-			o = v & PAGE_MASK;
-			c = min(uio->uio_resid, (u_int)(PAGE_SIZE - o));
-			vd = PHYS_TO_DMAP_RAW(v);
-			if (vd < DMAP_MIN_ADDRESS ||
-			    (vd > DMAP_MIN_ADDRESS + dmaplimit &&
-			    vd <= DMAP_MAX_ADDRESS) ||
-			    (pmap_kextract(vd) == 0 && (v & PG_FRAME) != 0)) {
-				error = EFAULT;
-				goto ret;
-			}
-			error = uiomove((void *)vd, (int)c, uio);
-			continue;
-		} else if (dev2unit(dev) == CDEV_MINOR_KMEM) {
-			v = uio->uio_offset;
-
-			if (v >= DMAP_MIN_ADDRESS && v < DMAP_MAX_ADDRESS) {
-				v = DMAP_TO_PHYS_RAW(v);
-				goto kmemphys;
-			}
-
-			c = iov->iov_len;
+		v = uio->uio_offset;
+		c = ulmin(iov->iov_len, PAGE_SIZE - (u_int)(v & PAGE_MASK));
 
+		switch (dev2unit(dev)) {
+		case CDEV_MINOR_KMEM:
 			/*
-			 * Make sure that all of the pages are currently
-			 * resident so that we don't create any zero-fill
-			 * pages.
+			 * Since c is clamped to be less or equal than
+			 * PAGE_SIZE, the uiomove() call does not
+			 * access past the end of the direct map.
 			 */
-			addr = trunc_page(v);
-			eaddr = round_page(v + c);
+			if (v >= DMAP_MIN_ADDRESS &&
+			    v < DMAP_MIN_ADDRESS + dmaplimit) {
+				error = uiomove((void *)v, c, uio);
+				break;
+			}
 
-			if (addr < VM_MIN_KERNEL_ADDRESS) {
-				error = EFAULT;
-				goto ret;
-			}
-			for (; addr < eaddr; addr += PAGE_SIZE) {
-				if (pmap_extract(kernel_pmap, addr) == 0) {
-					error = EFAULT;
-					goto ret;
-				}
-			}
-			if (!kernacc((caddr_t)(long)v, c,
-			    uio->uio_rw == UIO_READ ? 
+			if (!kernacc((void *)v, c, uio->uio_rw == UIO_READ ?
 			    VM_PROT_READ : VM_PROT_WRITE)) {
 				error = EFAULT;
-				goto ret;
+				break;
 			}
 
-			error = uiomove((caddr_t)(long)v, (int)c, uio);
-			continue;
+			/*
+			 * If the extracted address is not accessible
+			 * through the direct map, then we make a
+			 * private (uncached) mapping because we can't
+			 * depend on the existing kernel mapping
+			 * remaining valid until the completion of
+			 * uiomove().
+			 *
+			 * XXX We cannot provide access to the
+			 * physical page 0 mapped into KVA.
+			 */
+			v = pmap_extract(kernel_pmap, v);
+			if (v == 0) {
+				error = EFAULT;
+				break;
+			}
+			/* FALLTHROUGH */
+		case CDEV_MINOR_MEM:
+			if (v < dmaplimit) {
+				vd = PHYS_TO_DMAP(v);
+				error = uiomove((void *)vd, c, uio);
+				break;
+			}
+			if (v >= (1ULL << cpu_maxphyaddr)) {
+				error = EFAULT;
+				break;
+			}
+			p = pmap_mapdev(v, PAGE_SIZE);
+			error = uiomove(p, c, uio);
+			pmap_unmapdev((vm_offset_t)p, PAGE_SIZE);
+			break;
 		}
-		/* else panic! */
 	}
-ret:
 	curthread_pflags_restore(sflags);
+	/*
+	 * Don't return error if any byte was written.  Read and write
+	 * can return error only if no i/o was performed.
+	 */
+	if (uio->uio_resid != orig_resid)
+		error = 0;
 	return (error);
 }