mirror of
https://github.com/opnsense/src.git
synced 2026-06-11 01:30:30 -04:00
CI: Add full test support
The patch adds support for running full tests in the local CI Environment. New features added: - New target `ci-full` which runs the full test. This is also the default now - Renamed the previous target `ci-smokeit` to `ci-smoke` - Unlike previous if the available memory is more than 16G a default of 8G will be used - Removed some unnecessary debug messages - Added `dummybuf` kernel module to the list of modules to be loaded on the VM The features that can be tested: `make TARGET=<TARGET> TARGET_ARCH=<TARGET_ARCH> CITYPE=full ci` `make TARGET=<TARGET> TARGET_ARCH=<TARGET_ARCH> ci` is also the same as above as CITYPE full is the default now Approved by: lwhsu Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D48015
This commit is contained in:
Muhammad Moinur Rahman
parent
c201a1198a
commit
c08f5ad160
3 changed files with 79 additions and 24 deletions
|
|
@ -8,8 +8,9 @@
|
|||
# Makefile for CI testing.
|
||||
#
|
||||
# User-driven targets:
|
||||
# ci: Run CI tests. Currently only smoke tests are supported.
|
||||
# ci-smokeit: Currently same as ci.
|
||||
# ci: Run CI tests
|
||||
# ci-smoke: Run smoke tests which is simply booting the image
|
||||
# ci-full: Run full tests
|
||||
#
|
||||
# Variables affecting the build process:
|
||||
# TARGET/TARGET_ARCH: architecture of built release (default: same as build host)
|
||||
|
|
@ -68,11 +69,25 @@ SWAPSIZE?= 1g
|
|||
VMFS?= ufs
|
||||
FORMAT= raw
|
||||
CIIMAGE= ci-${OSRELEASE}-${GITREV}-${KERNCONF}.${FORMAT}
|
||||
CIDISK?= ${.OBJDIR}/${CIIMAGE}
|
||||
VMSIZE?= 6g
|
||||
CITYPE?=
|
||||
CITYPE?= full
|
||||
META_TAR!=mktemp /tmp/meta.XXXXXX
|
||||
META_DIR!=mktemp -d /tmp/meta.XXXXXX
|
||||
META_DIROUT!=mktemp -d /tmp/meta.XXXXXX
|
||||
DISC_CAM!=truncate -s 128m /tmp/disk-cam
|
||||
EXTRA_DISK_NUM?=5
|
||||
DISK_NUMBERS!=jot - 1 ${EXTRA_DISK_NUM}
|
||||
BHYVE_EXTRA_DISK_PARAM?=
|
||||
BHYVE_EXTRA_DISK_PARAM+=-s 4:0,ahci-hd,/tmp/disk-cam
|
||||
.for i in ${DISK_NUMBERS}
|
||||
NEW_DISK!=truncate -s 128m /tmp/disk${i}
|
||||
BHYVE_EXTRA_DISK_PARAM+=-s $$((${i} + 4)):0,virtio-blk,/tmp/disk${i}
|
||||
CLEANFILES+=/tmp/disk${i}
|
||||
.endfor
|
||||
TEST_VM_NAME= ci-${OSRELEASE}-${GITREV}-${KERNCONF}
|
||||
.if ${TOTAL_MEMORY} >= 16
|
||||
VM_MEM!=expr ${TOTAL_MEMORY} / 2
|
||||
VM_MEM=8
|
||||
.elif ${TOTAL_MEMORY} >=4
|
||||
VM_MEM=${TOTAL_MEMORY}
|
||||
.else
|
||||
|
|
@ -117,8 +132,8 @@ KLDFILEMONISLOADED!=kldload -q -n filemon 2>/dev/null && echo "1" || echo "0"
|
|||
METAMODE?=-DWITH_META_MODE
|
||||
.endif
|
||||
|
||||
CLEANFILES= ${CIIMAGE} ci.img
|
||||
CLEANDIRS= ci-buildimage
|
||||
CLEANFILES+= ${.OBJDIR}/${CIIMAGE} ${.OBJDIR}/ci.img ${META_TAR}
|
||||
CLEANDIRS+= ${.OBJDIR}/ci-buildimage
|
||||
|
||||
portinstall: portinstall-pkg portinstall-qemu portinstall-expect portinstall-${TARGET_ARCH:tl} .PHONY
|
||||
|
||||
|
|
@ -141,11 +156,13 @@ portinstall-expect: portinstall-pkg .PHONY
|
|||
.endif
|
||||
|
||||
beforeclean: .PHONY
|
||||
chflags -R noschg .
|
||||
chflags -R noschg ${.OBJDIR}/${.TARGET}
|
||||
|
||||
.include <bsd.obj.mk>
|
||||
clean: beforeclean .PHONY
|
||||
|
||||
cleandir: beforeclean .PHONY
|
||||
|
||||
ci-buildworld: .PHONY
|
||||
@echo "Building world for ${TARGET_ARCH}"
|
||||
${IMAKE} -j${PARALLEL_JOBS} -C ${WORLDDIR} ${METAMODE} \
|
||||
|
|
@ -155,7 +172,7 @@ ci-buildworld: .PHONY
|
|||
|
||||
|
||||
ci-buildkernel: ci-buildworld-${TARGET_ARCH:tl} .PHONY
|
||||
@echo "Building kenrel for ${TARGET_ARCH"}"
|
||||
@echo "Building kernel for ${TARGET_ARCH"}"
|
||||
${IMAKE} -j${PARALLEL_JOBS} -C ${WORLDDIR} ${METAMODE} \
|
||||
${CROSS_TOOLCHAIN_PARAM} __MAKE_CONF=${MAKECONF} \
|
||||
SRCCONF=${SRCCONF} buildkernel > ${.CURDIR}/_.${TARGET_ARCH}.${.TARGET} 2>&1 || \
|
||||
|
|
@ -174,19 +191,32 @@ ci-buildimage: ${QEMUTGT} ci-buildkernel-${TARGET_ARCH:tl} .PHONY
|
|||
(echo "${.TARGET} failed, check _.${TARGET_ARCH}.${.TARGET} for details" ; false)
|
||||
touch ${.TARGET}
|
||||
|
||||
ci-setsmokevar: .PHONY
|
||||
ci-set-smoke-var: .PHONY
|
||||
CITYPE=smoke
|
||||
|
||||
ci-set-full-var: .PHONY
|
||||
CITYPE=full
|
||||
|
||||
ci-create-meta: .PHONY
|
||||
truncate -s 512M ${META_TAR}
|
||||
tar rvf ${META_TAR} -C ${META_DIR} .
|
||||
|
||||
ci-extract-meta: .PHONY
|
||||
tar xfv ${META_TAR} -C ${META_DIROUT}
|
||||
@echo "Extracted kyua reports to ${META_DIROUT}"
|
||||
|
||||
ci-runtest: ci-buildimage-${TARGET_ARCH:tl} portinstall .PHONY
|
||||
.if ${MACHINE} == "amd64" && ( ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386" ) && ( !defined(USE_QEMU) || empty(USE_QEMU) )
|
||||
/usr/sbin/bhyvectl --vm=${TEST_VM_NAME} --destroy || true
|
||||
/usr/sbin/bhyveload -c stdio -m ${VM_MEM_SIZE} -d ${CIIMAGE} ${TEST_VM_NAME}
|
||||
/usr/sbin/bhyveload -c stdio -m ${VM_MEM_SIZE} -d ${CIDISK} ${TEST_VM_NAME}
|
||||
expect -c "set timeout ${TIMEOUT_EXPECT}; \
|
||||
spawn /usr/bin/timeout -k 60 ${TIMEOUT_VM} /usr/sbin/bhyve \
|
||||
-c ${PARALLEL_JOBS} -m ${VM_MEM_SIZE} -A -H -P \
|
||||
-s 0:0,hostbridge \
|
||||
-s 1:0,lpc \
|
||||
-s 2:0,virtio-blk,${CIIMAGE} \
|
||||
-s 2:0,virtio-blk,${CIDISK} \
|
||||
-s 3:0,virtio-blk,${META_TAR} \
|
||||
${BHYVE_EXTRA_DISK_PARAM} \
|
||||
-l com1,stdio \
|
||||
${TEST_VM_NAME}; \
|
||||
expect { eof }"
|
||||
|
|
@ -199,7 +229,8 @@ ci-runtest: ci-buildimage-${TARGET_ARCH:tl} portinstall .PHONY
|
|||
-nographic \
|
||||
-no-reboot \
|
||||
${QEMU_EXTRA_PARAM} \
|
||||
-drive if=none,file=${CIIMAGE},format=raw,id=hd0 \
|
||||
-drive if=none,file=${CIDISK},format=raw,id=hd0 \
|
||||
-drive if=none,file=${META_TAR},format=raw,id=hd1 \
|
||||
${QEMU_DEVICES}
|
||||
.endif
|
||||
|
||||
|
|
@ -215,8 +246,10 @@ ci-checktarget: .PHONY
|
|||
@echo "Error: ${TARGET_ARCH} is not supported on ${TYPE} ${REVISION} ${BRANCH}"
|
||||
.endif
|
||||
|
||||
ci-smokeit: ci-setsmokevar ci-checktarget .WAIT ci-runtest-${TARGET_ARCH:tl} .PHONY
|
||||
ci-smoke: ci-set-smoke-var ci-create-meta ci-checktarget .WAIT ci-runtest-${TARGET_ARCH:tl} .PHONY
|
||||
|
||||
ci: ci-smokeit .PHONY
|
||||
ci-full: ci-set-full-var ci-create-meta ci-checktarget .WAIT ci-runtest-${TARGET_ARCH:tl} ci-extract-meta .PHONY
|
||||
|
||||
ci: ci-${CITYPE:tl} .PHONY
|
||||
|
||||
.include "${RELEASEDIR}/Makefile.inc1"
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
export VM_RC_LIST="auditd freebsdci"
|
||||
|
||||
if [ "${CITYPE}" != "smoke" ]; then
|
||||
export VM_EXTRA_PACKAGES="coreutils devel/py-pytest gdb jq ksh93 net/py-dpkt net/scapy nist-kat nmap perl5 python python3 sudo tcptestsuite"
|
||||
export VM_EXTRA_PACKAGES="coreutils devel/py-pytest gdb jq ksh93 net/py-dpkt net/scapy nist-kat nmap perl5 python python3 sudo sysutils/porch tcptestsuite"
|
||||
|
||||
if [ "${TARGET}" = "amd64" ]; then
|
||||
export VM_EXTRA_PACKAGES="${VM_EXTRA_PACKAGES} linux-c7-ltp"
|
||||
|
|
@ -41,6 +41,7 @@ cat << EOF >> ${DESTDIR}/etc/rc.conf
|
|||
kld_list="" # Load modules needed by tests
|
||||
kld_list="${kld_list} blake2" # sys/opencrypto
|
||||
kld_list="${kld_list} cryptodev" # sys/opencrypto
|
||||
kld_list="${kld_list} dummymbuf" # sys/netpfil
|
||||
kld_list="${kld_list} fusefs" # sys/fs/fusefs
|
||||
kld_list="${kld_list} ipsec" # sys/netipsec
|
||||
kld_list="${kld_list} mac_portacl" # sys/mac/portacl
|
||||
|
|
@ -64,6 +65,10 @@ if [ "${CITYPE}" = "smoke" ]; then
|
|||
cat << EOF >> ${DESTDIR}/etc/rc.conf
|
||||
freebsdci_type="smoke"
|
||||
EOF
|
||||
elif [ "${CITYPE}" = "full" ]; then
|
||||
cat << EOF >> ${DESTDIR}/etc/rc.conf
|
||||
freebsdci_type="full"
|
||||
EOF
|
||||
fi
|
||||
cat << EOF >> ${DESTDIR}/etc/sysctl.conf
|
||||
kern.cryptodevallowsoft=1
|
||||
|
|
@ -77,7 +82,6 @@ cat << EOF >> ${DESTDIR}/etc/fstab
|
|||
fdesc /dev/fd fdescfs rw 0 0
|
||||
EOF
|
||||
mkdir -p ${DESTDIR}/usr/local/etc/rc.d
|
||||
echo $scriptdir
|
||||
cp -p ${scriptdir}/../../tests/ci/tools/freebsdci ${DESTDIR}/usr/local/etc/rc.d/
|
||||
touch ${DESTDIR}/firstboot
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@
|
|||
. /etc/rc.subr
|
||||
|
||||
: ${freebsdci_enable:="NO"}
|
||||
: ${freebsdci_type:="full"}
|
||||
|
||||
name="freebsdci"
|
||||
desc="Run FreeBSD CI"
|
||||
|
|
@ -33,10 +34,13 @@ rcvar=freebsdci_enable
|
|||
start_cmd="firstboot_ci_run"
|
||||
stop_cmd=":"
|
||||
os_arch=$(uname -p)
|
||||
tardev=/dev/vtbd1
|
||||
metadir=/meta
|
||||
istar=$(file -s ${tardev} | grep "POSIX tar archive" | wc -l)
|
||||
|
||||
auto_shutdown()
|
||||
{
|
||||
# XXX: Currently RISC-V kernels lack the ability to
|
||||
# NOTE: Currently RISC-V kernels lack the ability to
|
||||
# make qemu exit on shutdown. Reboot instead;
|
||||
# it makes qemu exit too.
|
||||
case "$os_arch" in
|
||||
|
|
@ -53,8 +57,6 @@ smoke_tests()
|
|||
{
|
||||
echo
|
||||
echo "--------------------------------------------------------------"
|
||||
echo "BUILD sequence COMPLETED"
|
||||
echo "IMAGE sequence COMPLETED"
|
||||
echo "BOOT sequence COMPLETED"
|
||||
echo "INITIATING system SHUTDOWN"
|
||||
echo "--------------------------------------------------------------"
|
||||
|
|
@ -62,15 +64,31 @@ smoke_tests()
|
|||
|
||||
full_tests()
|
||||
{
|
||||
# Currently this is a placeholder.
|
||||
# This will be used to add the full tests scenario those are run in
|
||||
# the CI system
|
||||
echo
|
||||
echo "--------------------------------------------------------------"
|
||||
echo "BUILD sequence COMPLETED"
|
||||
echo "IMAGE sequence COMPLETED"
|
||||
echo "BOOT sequence COMPLETED"
|
||||
echo "TEST sequence STARTED"
|
||||
if [ "${istar}" -eq 1 ]; then
|
||||
rm -fr ${metadir}
|
||||
mkdir -p ${metadir}
|
||||
tar xvf ${tardev} -C ${metadir}
|
||||
cd /usr/tests
|
||||
set +e
|
||||
kyua test
|
||||
rc=$?
|
||||
set -e
|
||||
if [ ${rc} -ne 0 ] && [ ${rc} -ne 1 ]; then
|
||||
exit ${rc}
|
||||
fi
|
||||
kyua report --verbose --results-filter passed,skipped,xfail,broken,failed --output test-report.txt
|
||||
kyua report-junit --output=test-report.xml
|
||||
mv test-report.* /${metadir}
|
||||
tar cvf ${tardev} -C ${metadir} .
|
||||
else
|
||||
echo "ERROR: no device with POSIX tar archive format found."
|
||||
# Don't shutdown because this is not run in unattended mode
|
||||
exit 1
|
||||
fi
|
||||
echo "TEST sequence COMPLETED"
|
||||
echo "INITIATING system SHUTDOWN"
|
||||
echo "--------------------------------------------------------------"
|
||||
|
|
|
|||
Loading…
Reference in a new issue