2011-05-13 00:54:01 -04:00
|
|
|
# $FreeBSD$
|
|
|
|
|
|
Initial support for bhyve save and restore.
Save and restore (also known as suspend and resume) permits a snapshot
to be taken of a guest's state that can later be resumed. In the
current implementation, bhyve(8) creates a UNIX domain socket that is
used by bhyvectl(8) to send a request to save a snapshot (and
optionally exit after the snapshot has been taken). A snapshot
currently consists of two files: the first holds a copy of guest RAM,
and the second file holds other guest state such as vCPU register
values and device model state.
To resume a guest, bhyve(8) must be started with a matching pair of
command line arguments to instantiate the same set of device models as
well as a pointer to the saved snapshot.
While the current implementation is useful for several uses cases, it
has a few limitations. The file format for saving the guest state is
tied to the ABI of internal bhyve structures and is not
self-describing (in that it does not communicate the set of device
models present in the system). In addition, the state saved for some
device models closely matches the internal data structures which might
prove a challenge for compatibility of snapshot files across a range
of bhyve versions. The file format also does not currently support
versioning of individual chunks of state. As a result, the current
file format is not a fixed binary format and future revisions to save
and restore will break binary compatiblity of snapshot files. The
goal is to move to a more flexible format that adds versioning,
etc. and at that point to commit to providing a reasonable level of
compatibility. As a result, the current implementation is not enabled
by default. It can be enabled via the WITH_BHYVE_SNAPSHOT=yes option
for userland builds, and the kernel option BHYVE_SHAPSHOT.
Submitted by: Mihai Tiganus, Flavius Anton, Darius Mihai
Submitted by: Elena Mihailescu, Mihai Carabas, Sergiu Weisz
Relnotes: yes
Sponsored by: University Politehnica of Bucharest
Sponsored by: Matthew Grooms (student scholarships)
Sponsored by: iXsystems
Differential Revision: https://reviews.freebsd.org/D19495
2020-05-04 20:02:04 -04:00
|
|
|
.include <kmod.opts.mk>
|
|
|
|
|
|
2011-05-13 00:54:01 -04:00
|
|
|
KMOD= vmm
|
|
|
|
|
|
Initial support for bhyve save and restore.
Save and restore (also known as suspend and resume) permits a snapshot
to be taken of a guest's state that can later be resumed. In the
current implementation, bhyve(8) creates a UNIX domain socket that is
used by bhyvectl(8) to send a request to save a snapshot (and
optionally exit after the snapshot has been taken). A snapshot
currently consists of two files: the first holds a copy of guest RAM,
and the second file holds other guest state such as vCPU register
values and device model state.
To resume a guest, bhyve(8) must be started with a matching pair of
command line arguments to instantiate the same set of device models as
well as a pointer to the saved snapshot.
While the current implementation is useful for several uses cases, it
has a few limitations. The file format for saving the guest state is
tied to the ABI of internal bhyve structures and is not
self-describing (in that it does not communicate the set of device
models present in the system). In addition, the state saved for some
device models closely matches the internal data structures which might
prove a challenge for compatibility of snapshot files across a range
of bhyve versions. The file format also does not currently support
versioning of individual chunks of state. As a result, the current
file format is not a fixed binary format and future revisions to save
and restore will break binary compatiblity of snapshot files. The
goal is to move to a more flexible format that adds versioning,
etc. and at that point to commit to providing a reasonable level of
compatibility. As a result, the current implementation is not enabled
by default. It can be enabled via the WITH_BHYVE_SNAPSHOT=yes option
for userland builds, and the kernel option BHYVE_SHAPSHOT.
Submitted by: Mihai Tiganus, Flavius Anton, Darius Mihai
Submitted by: Elena Mihailescu, Mihai Carabas, Sergiu Weisz
Relnotes: yes
Sponsored by: University Politehnica of Bucharest
Sponsored by: Matthew Grooms (student scholarships)
Sponsored by: iXsystems
Differential Revision: https://reviews.freebsd.org/D19495
2020-05-04 20:02:04 -04:00
|
|
|
SRCS= opt_acpi.h opt_bhyve_snapshot.h opt_ddb.h
|
|
|
|
|
SRCS+= device_if.h bus_if.h pci_if.h pcib_if.h acpi_if.h vnode_if.h
|
2017-12-05 12:23:33 -05:00
|
|
|
DPSRCS+= vmx_assym.h svm_assym.h
|
2018-07-03 17:02:25 -04:00
|
|
|
DPSRCS+= vmx_genassym.c svm_genassym.c offset.inc
|
2011-05-13 00:54:01 -04:00
|
|
|
|
2020-02-06 16:01:19 -05:00
|
|
|
CFLAGS+= -DVMM_KEEP_STATS
|
2017-03-04 05:10:17 -05:00
|
|
|
CFLAGS+= -I${SRCTOP}/sys/amd64/vmm
|
|
|
|
|
CFLAGS+= -I${SRCTOP}/sys/amd64/vmm/io
|
|
|
|
|
CFLAGS+= -I${SRCTOP}/sys/amd64/vmm/intel
|
|
|
|
|
CFLAGS+= -I${SRCTOP}/sys/amd64/vmm/amd
|
2011-05-13 00:54:01 -04:00
|
|
|
|
|
|
|
|
# generic vmm support
|
2017-03-04 05:10:17 -05:00
|
|
|
.PATH: ${SRCTOP}/sys/amd64/vmm
|
2011-05-13 00:54:01 -04:00
|
|
|
SRCS+= vmm.c \
|
|
|
|
|
vmm_dev.c \
|
2012-10-28 21:51:24 -04:00
|
|
|
vmm_host.c \
|
2012-09-25 18:31:35 -04:00
|
|
|
vmm_instruction_emul.c \
|
2014-03-11 12:56:00 -04:00
|
|
|
vmm_ioport.c \
|
2011-05-13 00:54:01 -04:00
|
|
|
vmm_lapic.c \
|
|
|
|
|
vmm_mem.c \
|
|
|
|
|
vmm_stat.c \
|
|
|
|
|
vmm_util.c \
|
2015-03-13 22:32:08 -04:00
|
|
|
x86.c
|
2011-05-13 00:54:01 -04:00
|
|
|
|
2017-03-04 05:10:17 -05:00
|
|
|
.PATH: ${SRCTOP}/sys/amd64/vmm/io
|
2011-05-13 00:54:01 -04:00
|
|
|
SRCS+= iommu.c \
|
|
|
|
|
ppt.c \
|
2014-03-11 12:56:00 -04:00
|
|
|
vatpic.c \
|
2014-03-25 15:20:34 -04:00
|
|
|
vatpit.c \
|
2013-11-25 14:04:51 -05:00
|
|
|
vhpet.c \
|
2013-11-12 17:51:03 -05:00
|
|
|
vioapic.c \
|
2014-10-26 00:44:28 -04:00
|
|
|
vlapic.c \
|
2014-12-30 17:19:34 -05:00
|
|
|
vpmtmr.c \
|
|
|
|
|
vrtc.c
|
2011-05-13 00:54:01 -04:00
|
|
|
|
|
|
|
|
# intel-specific files
|
2017-03-04 05:10:17 -05:00
|
|
|
.PATH: ${SRCTOP}/sys/amd64/vmm/intel
|
2011-05-13 00:54:01 -04:00
|
|
|
SRCS+= ept.c \
|
|
|
|
|
vmcs.c \
|
|
|
|
|
vmx_msr.c \
|
2014-10-17 09:20:49 -04:00
|
|
|
vmx_support.S \
|
2011-05-13 00:54:01 -04:00
|
|
|
vmx.c \
|
|
|
|
|
vtd.c
|
|
|
|
|
|
|
|
|
|
# amd-specific files
|
2017-03-04 05:10:17 -05:00
|
|
|
.PATH: ${SRCTOP}/sys/amd64/vmm/amd
|
2013-08-22 20:37:26 -04:00
|
|
|
SRCS+= vmcb.c \
|
AMD-vi: Fix IOMMU device interrupts being overridden
Currently, AMD-vi PCI-e passthrough will lead to the following lines in
dmesg:
"kernel: CPU0: local APIC error 0x40
ivhd0: Error: completion failed tail:0x720, head:0x0."
After some tracing, the problem is due to the interaction with
amdvi_alloc_intr_resources() and pci_driver_added(). In ivrs_drv, the
identification of AMD-vi IVHD is done by walking over the ACPI IVRS
table and ivhdX device_ts are added under the acpi bus, while there are
no driver handling the corresponding IOMMU PCI function. In
amdvi_alloc_intr_resources(), the MSI intr are allocated with the ivhdX
device_t instead of the IOMMU PCI function device_t. bus_setup_intr() is
called on ivhdX. the IOMMU pci function device_t is only used for
pci_enable_msi(). Since bus_setup_intr() is not called on IOMMU pci
function, the IOMMU PCI function device_t's dinfo->cfg.msi is never
updated to reflect the supposed msi_data and msi_addr. So the msi_data
and msi_addr stay in the value 0. When pci_driver_added() tried to loop
over the children of a pci bus, and do pci_cfg_restore() on each of
them, msi_addr and msi_data with value 0 will be written to the MSI
capability of the IOMMU pci function, thus explaining the errors in
dmesg.
This change includes an amdiommu driver which currently does attaching,
detaching and providing DEVMETHODs for setting up and tearing down
interrupt. The purpose of the driver is to prevent pci_driver_added()
from calling pci_cfg_restore() on the IOMMU PCI function device_t.
The introduction of the amdiommu driver handles allocation of an IRQ
resource within the IOMMU PCI function, so that the dinfo->cfg.msi is
populated.
This has been tested on EPYC Rome 7282 with Radeon 5700XT GPU.
Sponsored by: The FreeBSD Foundation
Reviewed by: jhb
Approved by: philip (mentor)
MFC after: 2 weeks
Differential Revision: https://reviews.freebsd.org/D28984
2021-03-22 05:33:43 -04:00
|
|
|
amdiommu.c \
|
|
|
|
|
ivhd_if.c \
|
|
|
|
|
ivhd_if.h \
|
2013-08-22 20:37:26 -04:00
|
|
|
svm.c \
|
2014-10-19 22:57:30 -04:00
|
|
|
svm_support.S \
|
2013-08-22 20:37:26 -04:00
|
|
|
npt.c \
|
2017-04-29 22:08:46 -04:00
|
|
|
ivrs_drv.c \
|
|
|
|
|
amdvi_hw.c \
|
2014-09-20 17:46:31 -04:00
|
|
|
svm_msr.c
|
2014-10-27 14:37:11 -04:00
|
|
|
|
Initial support for bhyve save and restore.
Save and restore (also known as suspend and resume) permits a snapshot
to be taken of a guest's state that can later be resumed. In the
current implementation, bhyve(8) creates a UNIX domain socket that is
used by bhyvectl(8) to send a request to save a snapshot (and
optionally exit after the snapshot has been taken). A snapshot
currently consists of two files: the first holds a copy of guest RAM,
and the second file holds other guest state such as vCPU register
values and device model state.
To resume a guest, bhyve(8) must be started with a matching pair of
command line arguments to instantiate the same set of device models as
well as a pointer to the saved snapshot.
While the current implementation is useful for several uses cases, it
has a few limitations. The file format for saving the guest state is
tied to the ABI of internal bhyve structures and is not
self-describing (in that it does not communicate the set of device
models present in the system). In addition, the state saved for some
device models closely matches the internal data structures which might
prove a challenge for compatibility of snapshot files across a range
of bhyve versions. The file format also does not currently support
versioning of individual chunks of state. As a result, the current
file format is not a fixed binary format and future revisions to save
and restore will break binary compatiblity of snapshot files. The
goal is to move to a more flexible format that adds versioning,
etc. and at that point to commit to providing a reasonable level of
compatibility. As a result, the current implementation is not enabled
by default. It can be enabled via the WITH_BHYVE_SNAPSHOT=yes option
for userland builds, and the kernel option BHYVE_SHAPSHOT.
Submitted by: Mihai Tiganus, Flavius Anton, Darius Mihai
Submitted by: Elena Mihailescu, Mihai Carabas, Sergiu Weisz
Relnotes: yes
Sponsored by: University Politehnica of Bucharest
Sponsored by: Matthew Grooms (student scholarships)
Sponsored by: iXsystems
Differential Revision: https://reviews.freebsd.org/D19495
2020-05-04 20:02:04 -04:00
|
|
|
.if ${KERN_OPTS:MBHYVE_SNAPSHOT} != ""
|
|
|
|
|
SRCS+= vmm_snapshot.c
|
|
|
|
|
.endif
|
|
|
|
|
|
2014-10-19 22:57:30 -04:00
|
|
|
CLEANFILES= vmx_assym.h vmx_genassym.o svm_assym.h svm_genassym.o
|
2011-05-13 00:54:01 -04:00
|
|
|
|
2017-12-05 12:23:33 -05:00
|
|
|
OBJS_DEPEND_GUESS.vmx_support.o+= vmx_assym.h
|
|
|
|
|
OBJS_DEPEND_GUESS.svm_support.o+= svm_assym.h
|
|
|
|
|
|
2014-10-17 09:20:49 -04:00
|
|
|
vmx_assym.h: vmx_genassym.o
|
2014-11-06 11:48:37 -05:00
|
|
|
sh ${SYSDIR}/kern/genassym.sh vmx_genassym.o > ${.TARGET}
|
2011-05-13 00:54:01 -04:00
|
|
|
|
2014-10-19 22:57:30 -04:00
|
|
|
svm_assym.h: svm_genassym.o
|
2014-11-06 11:48:37 -05:00
|
|
|
sh ${SYSDIR}/kern/genassym.sh svm_genassym.o > ${.TARGET}
|
2013-08-22 20:37:26 -04:00
|
|
|
|
2014-10-17 09:20:49 -04:00
|
|
|
vmx_support.o:
|
2011-05-13 00:54:01 -04:00
|
|
|
${CC} -c -x assembler-with-cpp -DLOCORE ${CFLAGS} \
|
|
|
|
|
${.IMPSRC} -o ${.TARGET}
|
|
|
|
|
|
2014-10-19 22:57:30 -04:00
|
|
|
svm_support.o:
|
2013-08-22 20:37:26 -04:00
|
|
|
${CC} -c -x assembler-with-cpp -DLOCORE ${CFLAGS} \
|
|
|
|
|
${.IMPSRC} -o ${.TARGET}
|
|
|
|
|
|
2018-07-03 17:02:25 -04:00
|
|
|
vmx_genassym.o: offset.inc
|
2020-04-18 08:54:40 -04:00
|
|
|
${CC} -c ${CFLAGS:N-flto:N-fno-common} -fcommon ${.IMPSRC}
|
2011-05-13 00:54:01 -04:00
|
|
|
|
2018-07-03 17:02:25 -04:00
|
|
|
svm_genassym.o: offset.inc
|
2020-04-18 08:54:40 -04:00
|
|
|
${CC} -c ${CFLAGS:N-flto:N-fno-common} -fcommon ${.IMPSRC}
|
2014-10-19 22:57:30 -04:00
|
|
|
|
2011-05-13 00:54:01 -04:00
|
|
|
.include <bsd.kmod.mk>
|
