refactor image-util script that builds e2e test-images

This commit is contained in:
upodroid 2026-01-23 19:38:33 +03:00
parent b2f73c0d6b
commit 7d4827cdba
22 changed files with 41 additions and 145 deletions

View file

@ -259,7 +259,7 @@ dependencies:
# GCB docker gcloud image
- name: "gcb-docker-gcloud: dependents"
version: v20240523-a15ad90fc9@sha256:bb04162508c2c61637eae700a0d8e8c8be8f2d4c831d2b75e59db2d4dd6cf75d
version: v20260108-7f313c340e@sha256:4d778a001ae3f4247b2b61d8a870319e71d66c87556969a6399b90972e4d0491
refPaths:
- path: build/pause/cloudbuild.yaml
match: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud

View file

@ -20,7 +20,7 @@ IMAGE = $(REGISTRY)/pause
TAG ?= 3.10.2
REV = $(shell git describe --contains --always --match='v*')
# Architectures supported: amd64, arm, arm64, ppc64le and s390x
# Architectures supported: amd64, arm64, ppc64le and s390x
ARCH ?= amd64
# Operating systems supported: linux, windows
OS ?= linux
@ -38,7 +38,7 @@ BASE := ${BASE.${OS}}
JQ_IMAGE := ghcr.io/jqlang/jq@sha256:a186dcd84a1e28bb48cdf3d7768b890d08621a87bb651fadb7db6815a6bf5ad5
ALL_OS = linux windows
ALL_ARCH.linux = amd64 arm arm64 ppc64le s390x
ALL_ARCH.linux = amd64 arm64 ppc64le s390x
ALL_OS_ARCH.linux = $(foreach arch, ${ALL_ARCH.linux}, linux-$(arch))
ALL_ARCH.windows = amd64
# ALL_OSVERSIONS lists all os.versions in BASE.windows.
@ -69,7 +69,6 @@ export DOCKER_CLI_EXPERIMENTAL=enabled
TRIPLE.windows-amd64 := x86_64-w64-mingw32
TRIPLE.linux-amd64 := x86_64-linux-gnu
TRIPLE.linux-arm := arm-linux-gnueabihf
TRIPLE.linux-arm64 := aarch64-linux-gnu
TRIPLE.linux-ppc64le := powerpc64le-linux-gnu
TRIPLE.linux-s390x := s390x-linux-gnu
@ -124,7 +123,7 @@ bin/wincat-windows-${ARCH}: windows/wincat/wincat.go
container: .container-${OS}-$(ARCH)
.container-linux-$(ARCH): bin/$(BIN)-$(OS)-$(ARCH)
docker buildx build --provenance=false --sbom=false --pull --output=type=${OUTPUT_TYPE} --platform ${OS}/$(ARCH) \
docker buildx build --pull --output=type=${OUTPUT_TYPE} --platform ${OS}/$(ARCH) \
-t $(IMAGE):$(TAG)-${OS}-$(ARCH) --build-arg BASE=${BASE} --build-arg ARCH=$(ARCH) .
touch $@

View file

@ -2,19 +2,13 @@
timeout: 1200s
options:
substitution_option: ALLOW_LOOSE
machineType: 'N1_HIGHCPU_8'
machineType: 'E2_HIGHCPU_8'
steps:
- name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20240523-a15ad90fc9@sha256:bb04162508c2c61637eae700a0d8e8c8be8f2d4c831d2b75e59db2d4dd6cf75d'
entrypoint: 'bash'
- name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20260108-7f313c340e@sha256:4d778a001ae3f4247b2b61d8a870319e71d66c87556969a6399b90972e4d0491'
dir: ./build/pause
env:
- DOCKER_CLI_EXPERIMENTAL=enabled
- REGISTRY=gcr.io/$PROJECT_ID
- IMAGE=gcr.io/$PROJECT_ID/pause
- HOME=/root
args:
- '-c'
- |
gcloud auth configure-docker \
&& docker buildx create --name img-builder --use \
&& make all-push
- make
- all-push

View file

@ -15,7 +15,6 @@
REPO_ROOT:=${CURDIR}/../..
REGISTRY ?= registry.k8s.io/e2e-test-images
DOCKER_CERT_BASE_PATH ?=
QEMUVERSION=v5.1.0-2
GOLANG_VERSION=$(shell cat $(REPO_ROOT)/.go-version)
export

View file

@ -12,12 +12,12 @@ new images, test the changes made, promote the newly built staging images.
## Prerequisites
In order to build the docker test images, a Linux node is required. The node will require `make`,
`docker (version 19.03.0 or newer)`, and ``docker buildx``, which will be used to build multiarch
`docker`, and ``docker buildx``, which will be used to build multiarch
images, as well as Windows images. In order to properly build multi-arch and Windows images, some
initialization is required (in CI this is done in [cloudbuild.yaml](cloudbuild.yaml)):
```shell
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
docker run --privileged --rm tonistiigi/binfmt --install all
docker buildx create --name img-builder --use
docker buildx inspect --bootstrap
```
@ -37,7 +37,7 @@ last known stable version.
Most tests used in E2E testing suite use the `agnhost` image. It contains several subcommands with
different [functionalities](agnhost/README.md) used to validate different Kubernetes behaviors. If
a new functionality needs testing, consider adding an `agnhost` subcommand for it first, before
a new functionality needs testing, add it as an `agnhost` subcommand first, before
creating an entirely separate test image.
The general process of making updates to the images is as follows:

View file

@ -29,8 +29,7 @@ RUN chmod +x /prepare_registry.sh
RUN /prepare_registry.sh
FROM $BASEIMAGE AS main
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
ARG TARGETARCH
# from dnsutils image
# install necessary packages:
@ -47,7 +46,7 @@ RUN apk --update add bind-tools curl netcat-openbsd iproute2 iperf bash util-lin
&& ln -s /usr/bin/iperf /usr/local/bin/iperf \
&& ls -altrh /usr/local/bin/iperf
ADD https://github.com/coredns/coredns/releases/download/v1.6.2/coredns_1.6.2_linux_BASEARCH.tgz /coredns.tgz
ADD https://github.com/coredns/coredns/releases/download/v1.6.2/coredns_1.6.2_linux_${TARGETARCH}.tgz /coredns.tgz
RUN tar -xzvf /coredns.tgz && rm -f /coredns.tgz
# PORT 80 needed by: test-webserver

View file

@ -1,4 +0,0 @@
linux/amd64=alpine:3.22
linux/arm64=arm64v8/alpine:3.22
linux/ppc64le=ppc64le/alpine:3.22
linux/s390x=s390x/alpine:3.22

View file

@ -12,11 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
ARG BASEIMAGE
ARG BASEIMAGE=alpine:3.22
FROM $BASEIMAGE
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
RUN apk add apparmor libapparmor --update-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ --allow-untrusted
ADD loader /usr/bin/loader

View file

@ -7,32 +7,17 @@ timeout: 5400s
# or any new substitutions added in the future.
options:
substitution_option: ALLOW_LOOSE
machineType: 'N1_HIGHCPU_8'
steps:
- name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20240523-a15ad90fc9@sha256:bb04162508c2c61637eae700a0d8e8c8be8f2d4c831d2b75e59db2d4dd6cf75d'
entrypoint: 'bash'
- name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20260108-7f313c340e@sha256:4d778a001ae3f4247b2b61d8a870319e71d66c87556969a6399b90972e4d0491'
dir: ./test/images/
env:
- DOCKER_CLI_EXPERIMENTAL=enabled
- BASE_REF=$_PULL_BASE_REF
- GIT_COMMIT_ID=$_PULL_BASE_SHA
- WHAT=$_WHAT
- REGISTRY=$_REGISTRY
# The default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx
# We need to set the HOME to /root explicitly since we're using docker buildx
- HOME=/root
# NOTE(claudiub): we need to call register.sh before creating and bootstraping a docker buildx instance.
args:
- '-c'
- |
gcloud auth configure-docker \
&& ../../third_party/multiarch/qemu-user-static/register/register.sh --reset -p yes \
&& export DOCKER_CLI_EXPERIMENTAL=enabled \
&& docker version \
&& docker buildx version \
&& docker buildx create --name img-builder --use \
&& docker buildx inspect --bootstrap \
&& make all-build-and-push
- make
- all-build-and-push
substitutions:
# _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
# can be used as a substitution
@ -46,3 +31,5 @@ substitutions:
_REGISTRY: 'gcr.io/k8s-staging-e2e-test-images'
# _WHAT will contain the image name to be built and published to the staging registry.
_WHAT: 'all-conformance'
tags:
- $_WHAT

View file

@ -27,8 +27,6 @@
ARG BASEIMAGE
FROM $BASEIMAGE
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
RUN apt-get update && \
apt-get install -y dnsutils && \
apt-get clean && \

View file

@ -21,20 +21,10 @@ set -o pipefail
TASK=${1}
WHAT=${2}
# docker buildx command is still experimental as of Docker 19.03.0
export DOCKER_CLI_EXPERIMENTAL="enabled"
# Connecting to a Remote Docker requires certificates for authentication, which can be found
# at this path. By default, they can be found in the ${HOME} folder. We're expecting to find
# here ".docker-${os_version}" folders which contains the necessary certificates.
DOCKER_CERT_BASE_PATH="${DOCKER_CERT_BASE_PATH:-${HOME}}"
KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
source "${KUBE_ROOT}/hack/lib/logging.sh"
source "${KUBE_ROOT}/hack/lib/util.sh"
# Mapping of go ARCH to actual architectures shipped part of multiarch/qemu-user-static project
declare -A QEMUARCHS=( ["amd64"]="x86_64" ["arm"]="arm" ["arm64"]="aarch64" ["ppc64le"]="ppc64le" ["s390x"]="s390x" )
declare -a DEFAULT_IMAGE_PLATFORMS=("linux/amd64" "linux/arm64" "linux/ppc64le" "linux/s390x")
# NOTE(claudiub): In the test image build jobs, this script is not being run in a git repository,
# which would cause git log to fail. Instead, we can use the GIT_COMMIT_ID set in cloudbuild.yaml.
@ -95,13 +85,12 @@ build() {
local image=${1}
local img_folder=${1}
local output_type=${2}
docker_version_check
local -a os_archs=()
if [[ -f "${img_folder}/BASEIMAGE" ]]; then
os_archs=$(listOsArchs "$image")
kube::util::read-array os_archs < <(listOsArchs "$image")
else
# prepend linux/ to the QEMUARCHS items.
os_archs=$(printf 'linux/%s\n' "${!QEMUARCHS[@]}")
os_archs=("${DEFAULT_IMAGE_PLATFORMS[@]}")
fi
# image tag
@ -116,7 +105,7 @@ build() {
kube::util::ensure-gnu-sed
kube::util::ensure-docker-buildx
for os_arch in ${os_archs}; do
for os_arch in "${os_archs[@]}"; do
splitOsArch "${image}" "${os_arch}"
if [[ "${os_name}" == "windows" && "${output_type}" == "docker" ]]; then
echo "Cannot build the image '${image}' for ${os_arch}. Built Windows container images need to be pushed to a registry."
@ -150,73 +139,31 @@ build() {
base_image=""
if [[ -f BASEIMAGE ]]; then
base_image=$(getBaseImage "${os_arch}" | "${SED}" "s|REGISTRY|${REGISTRY}|g")
"${SED}" -i "s|BASEARCH|${arch}|g" $dockerfile_name
fi
# Only the cross-build on x86 is guaranteed by far, other arches like aarch64 doesn't support cross-build
# thus, there is no need to tackle a disability feature on those platforms, and also help to prevent from
# ending up a wrong image tag on non-amd64 platforms.
build_arch=$(uname -m)
if [[ ${build_arch} = 'x86_64' ]]; then
# copy the qemu-*-static binary to docker image to build the multi architecture image on x86 platform
if grep -q 'CROSS_BUILD_' Dockerfile; then
if [[ "${arch}" = 'amd64' ]]; then
"${SED}" -i '/CROSS_BUILD_/d' Dockerfile
else
"${SED}" -i "s|QEMUARCH|${QEMUARCHS[$arch]}|g" Dockerfile
# Register qemu-*-static for all supported processors except the current one
echo 'Registering qemu-*-static binaries in the kernel'
local sudo=""
if [[ $(id -u) -ne 0 ]]; then
sudo="sudo"
fi
${sudo} docker run --rm --privileged tonistiigi/binfmt:latest --install all
curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/"${QEMUVERSION}"/x86_64_qemu-"${QEMUARCHS[$arch]}"-static.tar.gz | tar -xz -C "${temp_dir}"
# Ensure we don't get surprised by umask settings
chmod 0755 "${temp_dir}/qemu-${QEMUARCHS[$arch]}-static"
"${SED}" -i 's/CROSS_BUILD_//g' Dockerfile
fi
fi
elif [[ "${QEMUARCHS[$arch]}" != "${build_arch}" ]]; then
echo "skip cross-build $arch on non-supported platform ${build_arch}."
popd
continue
else
"${SED}" -i '/CROSS_BUILD_/d' Dockerfile
# If no BASEIMAGE file is found, extract base image from Dockerfile
# The build-arg can't be empty
base_image=$(sed -n 's/^ARG BASEIMAGE=//p' "${dockerfile_name}")
fi
# `--provenance=false --sbom=false` is set to avoid creating a manifest list: https://github.com/kubernetes/kubernetes/issues/123266
docker buildx build --progress=plain --no-cache --pull --output=type="${output_type}" --platform "${os_name}/${arch}" --provenance=false --sbom=false \
docker buildx build --progress=plain --no-cache --pull --output=type="${output_type}" --platform "${os_name}/${arch}" \
--build-arg BASEIMAGE="${base_image}" --build-arg REGISTRY="${REGISTRY}" --build-arg OS_VERSION="${os_version}" --build-arg GOLANG_VERSION="${GOLANG_VERSION}" \
-t "${REGISTRY}/${image}:${TAG}-${suffix}" -f "${dockerfile_name}" \
--label "image_version=${TAG}" --label "commit_id=${GIT_COMMIT_ID}" \
--label "git_url=https://github.com/kubernetes/kubernetes/tree/${GIT_COMMIT_ID}/test/images/${img_folder}" .
--label "image_version=${TAG}" --label "commit_id=${GIT_COMMIT_ID}" \
--label "git_url=https://github.com/kubernetes/kubernetes/tree/${GIT_COMMIT_ID}/test/images/${img_folder}" .
popd
done
}
docker_version_check() {
# docker manifest annotate --os-version has been introduced in 20.10.0,
# so we need to make sure we have it.
docker_version=$(docker version --format '{{.Client.Version}}' | cut -d"-" -f1)
if [[ ${docker_version} != 20.10.0 && ${docker_version} < 20.10.0 ]]; then
echo "Minimum docker version 20.10.0 is required for annotating the OS Version in the manifest list images: ${docker_version}]"
exit 1
fi
}
# This function will push the docker images
push() {
local image=${1}
docker_version_check
TAG=$(<"${image}"/VERSION)
local -a os_archs=()
if [[ -f ${image}/BASEIMAGE ]]; then
os_archs=$(listOsArchs "$image")
kube::util::read-array os_archs < <(listOsArchs "$image")
else
# prepend linux/ to the QEMUARCHS items.
os_archs=$(printf 'linux/%s\n' "${!QEMUARCHS[@]}")
os_archs=("${DEFAULT_IMAGE_PLATFORMS[@]}")
fi
pushd "${image}"
@ -231,10 +178,10 @@ push() {
# reset manifest list; needed in case multiple images are being built / pushed.
manifest=()
# Make os_archs list into image manifest. Eg: 'linux/amd64 linux/ppc64le' to '${REGISTRY}/${image}:${TAG}-linux-amd64 ${REGISTRY}/${image}:${TAG}-linux-ppc64le'
while IFS='' read -r line; do manifest+=("$line"); done < <(echo "$os_archs" | "${SED}" "s~\/~-~g" | "${SED}" -e "s~[^ ]*~$REGISTRY\/$image:$TAG\-&~g")
while IFS='' read -r line; do manifest+=("$line"); done < <(echo "${os_archs[@]}" | "${SED}" "s~\/~-~g" | "${SED}" -e "s~[^ ]*~$REGISTRY\/$image:$TAG\-&~g")
docker manifest create --amend "${REGISTRY}/${image}:${TAG}" "${manifest[@]}"
for os_arch in ${os_archs}; do
for os_arch in "${os_archs[@]}"; do
splitOsArch "${image}" "${os_arch}"
# For Windows images, we also need to include the "os.version" in the manifest list, so the Windows node

View file

@ -15,6 +15,4 @@
ARG BASEIMAGE
FROM $BASEIMAGE
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
RUN apk add --no-cache util-linux

View file

@ -13,9 +13,7 @@
# limitations under the License.
ARG BASEIMAGE
FROM $BASEIMAGE as build_node_perf_npb_ep
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
FROM $BASEIMAGE AS build_node_perf_npb_ep
RUN apt-get update && apt-get install -y libc6-dev g++ bzip2 dpkg-dev build-essential gfortran
@ -31,9 +29,9 @@ RUN sed -i '1i#!/bin/sh' sys/print_header sys/print_instructions
RUN if [ $(arch) != "x86_64" ]; then \
sed s/-mcmodel=medium//g config/NAS.samples/make.def_gcc > config/make.def; \
else \
else \
cp config/NAS.samples/make.def_gcc config/make.def; \
fi
fi
RUN make EP CLASS=D
# Copying the required libraries (shared object files) to a convenient location so that it can be copied into the

View file

@ -13,9 +13,7 @@
# limitations under the License.
ARG BASEIMAGE
FROM $BASEIMAGE as build_node_perf_npb_is
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
FROM $BASEIMAGE AS build_node_perf_npb_is
RUN apt-get update && apt-get install -y build-essential gfortran
@ -36,9 +34,9 @@ RUN sed -i '1i#!/bin/sh' sys/print_header sys/print_instructions
# -mno-outline-atomics: inline atomics to avoid linker issues
RUN if [ $(arch) = "aarch64" ]; then \
sed 's/-mcmodel=medium/-mcmodel=large -fno-PIE -mno-outline-atomics/g' config/NAS.samples/make.def.gcc_x86 > config/make.def; \
else \
else \
cp config/NAS.samples/make.def.gcc_x86 config/make.def; \
fi
fi
RUN make IS CLASS=D
# Copying the required libraries (shared object files) to a convenient location so that it can be copied into the

View file

@ -15,8 +15,6 @@
ARG BASEIMAGE
FROM $BASEIMAGE
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
# Install time, curl, and g++ (g++ required for torch.compile() inductor backend)
RUN apt-get update && apt-get install -y --no-install-recommends time curl g++ && \
rm -rf /var/lib/apt/lists/*

View file

@ -15,8 +15,6 @@
ARG BASEIMAGE
FROM $BASEIMAGE
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
COPY nnp /usr/local/bin/nnp
RUN chmod +s /usr/local/bin/nnp

View file

@ -15,8 +15,6 @@
ARG BASEIMAGE
FROM $BASEIMAGE
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
RUN clean-install wget bash dnsutils
COPY peer-finder /

View file

@ -17,8 +17,6 @@
ARG BASEIMAGE
FROM $BASEIMAGE
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
RUN clean-install wget bash netcat-openbsd
ADD on-start.sh /

View file

@ -15,8 +15,6 @@
ARG BASEIMAGE
FROM $BASEIMAGE
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
ADD regression-issue-74839 /regression-issue-74839
ENTRYPOINT ["/regression-issue-74839"]

View file

@ -15,8 +15,6 @@
ARG BASEIMAGE
FROM $BASEIMAGE
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
RUN clean-install stress
ADD consumer /consumer

View file

@ -15,8 +15,6 @@
ARG BASEIMAGE
FROM $BASEIMAGE
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
RUN yum install -y targetcli && yum clean all
ADD run_iscsi_target.sh /usr/local/bin/
ADD block.tar.gz /

View file

@ -15,7 +15,6 @@
ARG BASEIMAGE
FROM $BASEIMAGE
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
RUN dnf -y install procps-ng nfs-utils && dnf clean all
RUN mkdir -p /exports
ADD run_nfs.sh /usr/local/bin/