From 9be40761d786ff4d54cbf6c80ee9bdd0b39396d1 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Wed, 23 Oct 2024 09:36:06 -0700 Subject: [PATCH] Reduce the number of GH api request for E2E nightly (#11148) Signed-off-by: Derek Nola --- tests/e2e/scripts/latest_commit.sh | 24 ++++++++++++++++++++++-- tests/e2e/scripts/run_tests.sh | 27 ++++++++++++++++++++++++++- tests/e2e/vagrantdefaults.rb | 11 +++++++++-- 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/tests/e2e/scripts/latest_commit.sh b/tests/e2e/scripts/latest_commit.sh index 65b68e36eaa..787bc11df1a 100755 --- a/tests/e2e/scripts/latest_commit.sh +++ b/tests/e2e/scripts/latest_commit.sh @@ -4,8 +4,6 @@ branch=$1 output_file=$2 # Grabs the last 10 commit SHA's from the given branch, then purges any commits that do not have a passing CI build iterations=0 -commits_str=$(curl -s -H 'Accept: application/vnd.github.v3+json' "https://api.github.com/repos/k3s-io/k3s/commits?per_page=10&sha=$branch" | jq -j -r '.[] | .sha, " "') -read -a commits <<< "$commits_str" # The VMs take time on startup to hit aws, wait loop until we can while ! curl -s --fail https://k3s-ci-builds.s3.amazonaws.com > /dev/null; do @@ -17,6 +15,28 @@ while ! curl -s --fail https://k3s-ci-builds.s3.amazonaws.com > /dev/null; do sleep 1 done +if [ -n "$GH_TOKEN" ]; then + response=$(curl -s -H "Authorization: token $GH_TOKEN" -H 'Accept: application/vnd.github.v3+json' "https://api.github.com/repos/k3s-io/k3s/commits?per_page=10&sha=$branch") +else + response=$(curl -s -H 'Accept: application/vnd.github.v3+json' "https://api.github.com/repos/k3s-io/k3s/commits?per_page=10&sha=$branch") +fi +type=$(echo "$response" | jq -r type) + +# Verify if the response is an array with the k3s commits +if [[ $type == "object" ]]; then + message=$(echo "$response" | jq -r .message) + if [[ $message == "API rate limit exceeded for "* ]]; then + echo "Github API rate limit exceeded" + exit 1 + fi + echo "Github API returned a non-expected response ${message}" + exit 1 +elif [[ $type == "array" ]]; then + commits_str=$(echo "$response" | jq -j -r '.[] | .sha, " "') +fi + +read -a commits <<< "$commits_str" + for commit in "${commits[@]}"; do if curl -s --fail https://k3s-ci-builds.s3.amazonaws.com/k3s-$commit.sha256sum > /dev/null; then echo "$commit" > "$output_file" diff --git a/tests/e2e/scripts/run_tests.sh b/tests/e2e/scripts/run_tests.sh index d26b732599a..a30b6b478ee 100755 --- a/tests/e2e/scripts/run_tests.sh +++ b/tests/e2e/scripts/run_tests.sh @@ -6,6 +6,26 @@ agentcount=${3:-1} db=${4:-"etcd"} hardened=${5:-""} +cleanup() { + for net in $(virsh net-list --all | tail -n +2 | tr -s ' ' | cut -d ' ' -f2 | grep -v default); do + virsh net-destroy "$net" + virsh net-undefine "$net" + done + + for domain in $(virsh list --all | tail -n +2 | tr -s ' ' | cut -d ' ' -f3); do + virsh destroy "$domain" + virsh undefine "$domain" --remove-all-storage + done + + for vm in $(vagrant global-status |tr -s ' '|tail +3 |grep "/" |cut -d ' ' -f5); do + cd $vm + vagrant destroy -f + cd .. + done + # Prune Vagrant global status + vagrant global-status --prune +} + E2E_EXTERNAL_DB=$db && export E2E_EXTERNAL_DB E2E_REGISTRY=true && export E2E_REGISTRY @@ -18,7 +38,10 @@ OS=$(echo "$nodeOS"|cut -d'/' -f2) echo "$OS" vagrant global-status | awk '/running/'|cut -c1-7| xargs -r -d '\n' -n 1 -- vagrant destroy -f -E2E_RELEASE_CHANNEL="commit" && export E2E_RELEASE_CHANNEL + +# To reduce GH API requsts, we grab the latest commit on the host and pass it to the tests +./scripts/latest_commit.sh master latest_commit.txt +E2E_RELEASE_VERSION=$(cat latest_commit.txt) && export E2E_RELEASE_VERSION echo 'RUNNING DUALSTACK TEST' E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v dualstack/dualstack_test.go -nodeOS="$nodeOS" -serverCount=1 -agentCount=1 -timeout=30m -json -ci |tee k3s_"$OS".log @@ -44,6 +67,8 @@ echo 'RUNNING SNAPSHOT AND RESTORE TEST' echo 'RUNNING ROTATE CUSTOM CA TEST' /usr/local/go/bin/go test -v rotateca/rotateca_test.go -nodeOS="$nodeOS" -serverCount=1 -agentCount=1 -timeout=30m -json -ci | tee -a k3s_"$OS".log +# For upgrade test we use the release channel install as the starting point +unset E2E_RELEASE_VERSION E2E_RELEASE_CHANNEL="latest" && export E2E_RELEASE_CHANNEL echo 'RUNNING CLUSTER UPGRADE TEST' E2E_REGISTRY=true /usr/local/go/bin/go test -v upgradecluster/upgradecluster_test.go -nodeOS="$nodeOS" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=1h -json -ci | tee -a k3s_"$OS".log diff --git a/tests/e2e/vagrantdefaults.rb b/tests/e2e/vagrantdefaults.rb index 5e57e022bc1..3d9fc65954d 100644 --- a/tests/e2e/vagrantdefaults.rb +++ b/tests/e2e/vagrantdefaults.rb @@ -14,11 +14,18 @@ def defaultOSConfigure(vm) end end +# getInstallType is used to control which version of k3s to install +# To install a specific version, set release_version to the version number +# To install a specific commit, set release_version to the commit SHA +# To install the latest commit from a branch, leave release_version empty +# and set release_channel to "commit" and set branch to the branch name def getInstallType(vm, release_version, branch, release_channel='') if release_version == "skip" install_type = "INSTALL_K3S_SKIP_DOWNLOAD=true" - elsif !release_version.empty? + elsif !release_version.empty? && release_version.start_with?("v1") return "INSTALL_K3S_VERSION=#{release_version}" + elsif !release_version.empty? + return "INSTALL_K3S_COMMIT=#{release_version}" elsif !release_channel.empty? && release_channel != "commit" return "INSTALL_K3S_CHANNEL=#{release_channel}" else @@ -26,7 +33,7 @@ def getInstallType(vm, release_version, branch, release_channel='') scripts_location = Dir.exist?("./scripts") ? "./scripts" : "../scripts" # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build # MicroOS requires it not be in a /tmp/ or other root system folder - vm.provision "Get latest commit", type: "shell", path: scripts_location +"/latest_commit.sh", args: [branch, "/tmp/k3s_commits"] + vm.provision "Get latest commit", type: "shell", path: scripts_location +"/latest_commit.sh", env: {GH_TOKEN:ENV['GH_TOKEN']}, args: [branch, "/tmp/k3s_commits"] return "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /tmp/k3s_commits)" end end