From 461663ddbad02a4a5135673d545695b1a9f25ed0 Mon Sep 17 00:00:00 2001
From: John Baldwin <jhb@FreeBSD.org>
Date: Wed, 21 Dec 2022 10:31:16 -0800
Subject: [PATCH] bhyve: Simplify setting vCPU capabilities.

- Enable VM_CAP_IPI_EXIT in fbsdrun_set_capabilities along with other
  capabilities enabled on all vCPUs.

- Don't call fbsdrun_set_capabilities a second time on the BSP in
  spinup_vcpu.

- To preserve previous behavior, don't unconditionally enable
  unrestricted guest mode on the BSP (this unbreaks single-vCPU guests
  on Nehalem systems, though supporting such setups is of dubious
  value).  Other places that enbale UG on the BSP are careful to check
  the result of the operation and fail if it is not available.

- Don't set any capabilities in spinup_ap().  These are now all
  redundant with earlier settings from spinup_vcpu().

- While here, axe a stale comment from fbsdrun_addcpu().  This
  function is now always called from the main thread for all vCPUs.

Reviewed by:	corvink, markj
Differential Revision:	https://reviews.freebsd.org/D37642
---
 usr.sbin/bhyve/bhyverun.c  | 30 ++++++++++++++++--------------
 usr.sbin/bhyve/bhyverun.h  |  1 -
 usr.sbin/bhyve/spinup_ap.c | 14 --------------
 3 files changed, 16 insertions(+), 29 deletions(-)

diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index 8bc41950477..72f806e9799 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -552,12 +552,6 @@ fbsdrun_addcpu(struct vmctx *ctx, int newcpu, uint64_t rip, bool suspend)
 {
 	int error;
 
-	/*
-	 * The 'newcpu' must be activated in the context of 'fromcpu'. If
-	 * vm_activate_cpu() is delayed until newcpu's pthread starts running
-	 * then vmm.ko is out-of-sync with bhyve and this can create a race
-	 * with vm_suspend().
-	 */
 	error = vm_activate_cpu(ctx, newcpu);
 	if (error != 0)
 		err(EX_OSERR, "could not activate CPU %d", newcpu);
@@ -1044,7 +1038,7 @@ num_vcpus_allowed(struct vmctx *ctx)
 		return (1);
 }
 
-void
+static void
 fbsdrun_set_capabilities(struct vmctx *ctx, int cpu)
 {
 	int err, tmp;
@@ -1086,6 +1080,9 @@ fbsdrun_set_capabilities(struct vmctx *ctx, int cpu)
 	}
 
 	vm_set_capability(ctx, cpu, VM_CAP_ENABLE_INVPCID, 1);
+
+	err = vm_set_capability(ctx, cpu, VM_CAP_IPI_EXIT, 1);
+	assert(err == 0);
 }
 
 static struct vmctx *
@@ -1157,16 +1154,21 @@ spinup_vcpu(struct vmctx *ctx, int vcpu, bool suspend)
 	int error;
 	uint64_t rip;
 
+	if (vcpu != BSP) {
+		fbsdrun_set_capabilities(ctx, vcpu);
+
+		/*
+		 * Enable the 'unrestricted guest' mode for APs.
+		 *
+		 * APs startup in power-on 16-bit mode.
+		 */
+		error = vm_set_capability(ctx, vcpu, VM_CAP_UNRESTRICTED_GUEST, 1);
+		assert(error == 0);
+	}
+
 	error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RIP, &rip);
 	assert(error == 0);
 
-	fbsdrun_set_capabilities(ctx, vcpu);
-	error = vm_set_capability(ctx, vcpu, VM_CAP_UNRESTRICTED_GUEST, 1);
-	assert(error == 0);
-
-	error = vm_set_capability(ctx, vcpu, VM_CAP_IPI_EXIT, 1);
-	assert(error == 0);
-
 	fbsdrun_addcpu(ctx, vcpu, rip, suspend);
 }
 
diff --git a/usr.sbin/bhyve/bhyverun.h b/usr.sbin/bhyve/bhyverun.h
index 1e1e5f9880f..d3eb8c8b23d 100644
--- a/usr.sbin/bhyve/bhyverun.h
+++ b/usr.sbin/bhyve/bhyverun.h
@@ -45,7 +45,6 @@ void *paddr_guest2host(struct vmctx *ctx, uintptr_t addr, size_t len);
 uintptr_t paddr_host2guest(struct vmctx *ctx, void *addr);
 #endif
 
-void fbsdrun_set_capabilities(struct vmctx *ctx, int cpu);
 int  fbsdrun_virtio_msix(void);
 
 int vmexit_task_switch(struct vmctx *, struct vm_exit *, int *vcpu);
diff --git a/usr.sbin/bhyve/spinup_ap.c b/usr.sbin/bhyve/spinup_ap.c
index e23265f8b82..af8d40030a0 100644
--- a/usr.sbin/bhyve/spinup_ap.c
+++ b/usr.sbin/bhyve/spinup_ap.c
@@ -87,20 +87,6 @@ spinup_ap(struct vmctx *ctx, int newcpu, uint64_t rip)
 	error = vcpu_reset(ctx, newcpu);
 	assert(error == 0);
 
-	fbsdrun_set_capabilities(ctx, newcpu);
-
-	/*
-	 * Enable the 'unrestricted guest' mode for 'newcpu'.
-	 *
-	 * Set up the processor state in power-on 16-bit mode, with the CS:IP
-	 * init'd to the specified low-mem 4K page.
-	 */
-	error = vm_set_capability(ctx, newcpu, VM_CAP_UNRESTRICTED_GUEST, 1);
-	assert(error == 0);
-
-	error = vm_set_capability(ctx, newcpu, VM_CAP_IPI_EXIT, 1);
-	assert(error == 0);
-
 	spinup_ap_realmode(ctx, newcpu, &rip);
 
 	vm_resume_cpu(ctx, newcpu);