From 925ecb0aae7caa257c8c704364f715e463c44f47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Wed, 11 Dec 2019 12:04:29 +0100 Subject: [PATCH 1/7] List paths which should be excluded from tarballs Convert the logic (currently present in the form of "rm -rf" calls in util/kit.sh) for removing files and directories which are tracked by Git but redundant in release tarballs into a set of .gitattributes rules which allow the same effect to be achieved using "git archive". --- .gitattributes | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitattributes b/.gitattributes index 9da70ba2fc..3dac3b0fe6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,10 @@ *.sln.in eol=crlf *.vcxproj.* eol=crlf + +.gitignore export-ignore +/conftools export-ignore +/doc/design export-ignore +/doc/dev export-ignore +/util/** export-ignore +/util/bindkeys.pl -export-ignore +/util/mksymtbl.pl -export-ignore From c0be772ebc1406b761eb870d97acd72ebebad541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Wed, 11 Dec 2019 12:04:29 +0100 Subject: [PATCH 2/7] Include prepare-softhsm2.sh in source tarballs The util/prepare-softhsm2.sh script is useful for initializing a working SoftHSM environment which can be used by unit tests and system tests. However, since it is a test-specific script, it does not really belong in the util/ subdirectory which is mostly pruned during the BIND source tarball creation process. Move the prepare-softhsm2.sh script to bin/tests/ so that its location is more appropriate for its purpose and also so that it does not get removed during the BIND source tarball creation process, allowing it to be used for setting up test environments for tarball-based builds. --- .gitlab-ci.yml | 2 +- {util => bin/tests}/prepare-softhsm2.sh | 0 bin/tests/system/runtime/tests.sh | 2 +- util/copyrights | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename {util => bin/tests}/prepare-softhsm2.sh (100%) mode change 100644 => 100755 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 654074c5b4..c72d8b52ea 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -202,7 +202,7 @@ stages: fi .setup_softhsm: &setup_softhsm | - sh -x util/prepare-softhsm2.sh + sh -x bin/tests/prepare-softhsm2.sh .system_test: &system_test_job <<: *default_triggering_rules diff --git a/util/prepare-softhsm2.sh b/bin/tests/prepare-softhsm2.sh old mode 100644 new mode 100755 similarity index 100% rename from util/prepare-softhsm2.sh rename to bin/tests/prepare-softhsm2.sh diff --git a/bin/tests/system/runtime/tests.sh b/bin/tests/system/runtime/tests.sh index e43e96e0ba..e35cb70267 100644 --- a/bin/tests/system/runtime/tests.sh +++ b/bin/tests/system/runtime/tests.sh @@ -204,7 +204,7 @@ if [ "$(id -u)" -eq 0 ] && [ -z "$CYGWIN" ]; then if [ "$?" -eq 0 ]; then copy_setports ns2/named-alt9.conf.in "${TEMP_NAMED_DIR}/named-alt9.conf" export SOFTHSM2_CONF="${TEMP_NAMED_DIR}/softhsm2.conf" - sh "$TOP/util/prepare-softhsm2.sh" + sh "$TOP/bin/tests/prepare-softhsm2.sh" chown -R nobody: "${TEMP_NAMED_DIR}" chmod 0700 "${TEMP_NAMED_DIR}" ( cd "${TEMP_NAMED_DIR}" && $NAMED -u nobody -c named-alt9.conf -d 99 -g -U 4 >> named$n.run 2>&1 & ) || ret=1 diff --git a/util/copyrights b/util/copyrights index a14a28c124..d8e7538431 100644 --- a/util/copyrights +++ b/util/copyrights @@ -349,6 +349,7 @@ ./bin/tests/pkcs11/benchmarks/sha1.c C 2014,2015,2016,2018,2019 ./bin/tests/pkcs11/benchmarks/sign.c C 2014,2015,2016,2018,2019 ./bin/tests/pkcs11/benchmarks/verify.c C 2014,2015,2016,2018,2019 +./bin/tests/prepare-softhsm2.sh X 2018,2019 ./bin/tests/startperf/README X 2011,2018,2019 ./bin/tests/startperf/clean.sh SH 2011,2012,2016,2018,2019 ./bin/tests/startperf/makenames.pl PERL 2011,2012,2016,2018,2019 @@ -2592,7 +2593,6 @@ ./util/nanny.pl PERL 2000,2001,2004,2007,2012,2016,2018,2019 ./util/new-func PERL 2005,2007,2012,2016,2018,2019 ./util/nt-kit SH 1999,2000,2001,2004,2007,2012,2016,2018,2019 -./util/prepare-softhsm2.sh X 2018,2019 ./util/spacewhack.pl PERL 2000,2001,2004,2007,2012,2016,2018,2019 ./util/tabify-changes SH 2004,2007,2012,2016,2018,2019 ./util/update-drafts.pl PERL 2000,2001,2004,2007,2012,2016,2018,2019 From 8d567490460c2ef6b00bc505793f31c06718fee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Wed, 11 Dec 2019 12:04:29 +0100 Subject: [PATCH 3/7] Create and test BIND source tarballs in GitLab CI Add a set of jobs to GitLab CI that create a BIND source tarball and then build and test its contents. Run those extra jobs only when a tag is pushed to the Git repository as they are only meant to be sanity checks of BIND source tarball contents. --- .gitlab-ci.yml | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c72d8b52ea..60587ff5f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,6 +21,9 @@ variables: CFLAGS_COMMON: -fno-omit-frame-pointer -fno-optimize-sibling-calls -O1 -g -Wall -Wextra + TARBALL_COMPRESSOR: xz + TARBALL_EXTENSION: xz + stages: - precheck - build @@ -318,6 +321,23 @@ misc:sid:amd64: - util/check-cocci - if test "$(git status --porcelain | grep -Ev '\?\?' | wc -l)" -gt "0"; then git status --short; exit 1; fi +tarball-create:sid:amd64: + <<: *debian_sid_amd64_image + stage: precheck + script: + - source version + - export BIND_DIRECTORY="bind-${MAJORVER}.${MINORVER}.${PATCHVER}${RELEASETYPE}${RELEASEVER}" + - git archive --prefix="${BIND_DIRECTORY}/" --output="${BIND_DIRECTORY}.tar" HEAD + - mkdir "${BIND_DIRECTORY}" + - echo "SRCID=$(git rev-list --max-count=1 HEAD | cut -b1-7)" > "${BIND_DIRECTORY}/srcid" + - tar --append --file="${BIND_DIRECTORY}.tar" "${BIND_DIRECTORY}/srcid" + - ${TARBALL_COMPRESSOR} "${BIND_DIRECTORY}.tar" + artifacts: + paths: + - bind-*.tar.${TARBALL_EXTENSION} + only: + - tags + # Jobs for doc builds on Debian Sid (amd64) docs:sid:amd64: @@ -578,6 +598,48 @@ cppcheck:gcc:sid:amd64: - gcc:sid:amd64 needs: ["gcc:sid:amd64"] +# Jobs for tarball GCC builds on Debian Sid (amd64) + +tarball:sid:amd64: + variables: + CC: gcc + EXTRA_CONFIGURE: "--enable-dnstap --with-libidn2" + RUN_MAKE_INSTALL: 1 + <<: *debian_sid_amd64_image + <<: *build_job + before_script: + - tar --extract --file bind-*.tar.${TARBALL_EXTENSION} + - rm -f bind-*.tar.${TARBALL_EXTENSION} + - cd bind-* + dependencies: + - tarball-create:sid:amd64 + needs: ["tarball-create:sid:amd64"] + only: + - tags + +system:tarball:sid:amd64: + <<: *debian_sid_amd64_image + <<: *system_test_job + before_script: + - cd bind-* + - *setup_interfaces + dependencies: + - tarball:sid:amd64 + needs: ["tarball:sid:amd64"] + only: + - tags + +unit:tarball:sid:amd64: + <<: *debian_sid_amd64_image + <<: *unit_test_job + before_script: + - cd bind-* + dependencies: + - tarball:sid:amd64 + needs: ["tarball:sid:amd64"] + only: + - tags + # Jobs for regular GCC builds on Debian Sid (i386) gcc:sid:i386: From 12564928a76f8ff613ac061e243693d29279339d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Wed, 11 Dec 2019 12:04:29 +0100 Subject: [PATCH 4/7] Add a Windows debug build job to GitLab CI Add a Visual Studio build job using the "Debug" build configuration to GitLab CI without enabling it for every pipeline as it takes about twice as long to complete as its "Release" counterpart. --- .gitlab-ci.yml | 68 +++++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 60587ff5f4..2b54242a26 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -197,6 +197,36 @@ stages: untracked: true expire_in: "1 week" +.windows_build: &windows_build_job + stage: build + tags: + - windows + - amd64 + script: + - 'Push-Location "C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/VC/Auxiliary/Build"' + - '& cmd.exe /C "vcvarsall.bat x64 & set" | Foreach-Object { if ($_ -match "(.*?)=(.*)") { Set-Item -force -path "Env:\$($matches[1])" -value "$($matches[2])" } }' + - 'Pop-Location' + - 'Set-Location win32utils' + - '& "C:/Strawberry/perl/bin/perl.exe" Configure + "with-tools-version=15.0" + "with-platform-toolset=v141" + "with-platform-version=10.0.17763.0" + "with-vcredist=C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/VC/Redist/MSVC/14.16.27012/vcredist_x64.exe" + "with-openssl=C:/OpenSSL" + "with-libxml2=C:/libxml2" + "with-libuv=C:/libuv" + "without-python" + "with-system-tests" + x64' + - 'Set-Item -path "Env:CL" -value "/MP$([Math]::Truncate($BUILD_PARALLEL_JOBS/2))"' + - '& msbuild.exe /maxCpuCount:2 /t:Build /p:Configuration=$VSCONF bind9.sln' + dependencies: [] + needs: + - autoreconf:sid:amd64 + artifacts: + untracked: true + expire_in: "1 week" + .setup_interfaces: &setup_interfaces | if [ "$(id -u)" -eq "0" ]; then sh -x bin/tests/system/ifconfig.sh up; @@ -949,37 +979,10 @@ unit:nolibtool:sid:amd64: # Jobs for Visual Studio 2017 builds on Windows (amd64) msvc:windows:amd64: + <<: *windows_build_job <<: *default_triggering_rules - stage: build - tags: - - windows - - amd64 variables: VSCONF: Release - script: - - 'Push-Location "C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/VC/Auxiliary/Build"' - - '& cmd.exe /C "vcvarsall.bat x64 & set" | Foreach-Object { if ($_ -match "(.*?)=(.*)") { Set-Item -force -path "Env:\$($matches[1])" -value "$($matches[2])" } }' - - 'Pop-Location' - - 'Set-Location win32utils' - - '& "C:/Strawberry/perl/bin/perl.exe" Configure - "with-tools-version=15.0" - "with-platform-toolset=v141" - "with-platform-version=10.0.17763.0" - "with-vcredist=C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/VC/Redist/MSVC/14.16.27012/vcredist_x64.exe" - "with-openssl=C:/OpenSSL" - "with-libxml2=C:/libxml2" - "with-libuv=C:/libuv" - "without-python" - "with-system-tests" - x64' - - 'Set-Item -path "Env:CL" -value "/MP$([Math]::Truncate($BUILD_PARALLEL_JOBS/2))"' - - '& msbuild.exe /maxCpuCount:2 /t:Build /p:Configuration=$VSCONF bind9.sln' - dependencies: [] - needs: - - autoreconf:sid:amd64 - artifacts: - untracked: true - expire_in: "1 week" system:msvc:windows:amd64: stage: system @@ -1009,3 +1012,12 @@ system:msvc:windows:amd64: - schedules - tags - web + +msvc-debug:windows:amd64: + <<: *windows_build_job + variables: + VSCONF: Debug + only: + - schedules + - tags + - web From 2b1c8c54d1e88c0d70accb4e85e0948c04478be9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Wed, 11 Dec 2019 12:04:29 +0100 Subject: [PATCH 5/7] Add a Windows debug system test job to GitLab CI Add a system test job for binaries created by Visual Studio in the "Debug" build configuration to GitLab CI so that they can be tested along their "Release" counterparts when necessary. --- .gitlab-ci.yml | 55 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b54242a26..b947489952 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -258,6 +258,30 @@ stages: --results-filter "" \ --output kyua_html +.windows_system_test: &windows_system_test_job + stage: system + tags: + - windows + - amd64 + script: + - 'Push-Location bin/tests/system' + - '$ifIndex = Get-NetIPInterface -AddressFamily IPv4 -InterfaceMetric 75 | Select-Object -ExpandProperty ifIndex' + - '& C:/tools/cygwin/bin/sed.exe -i "s/^exit.*/netsh interface ipv4 set dnsservers $ifIndex dhcp/; s/\(name\|interface\)=Loopback/$ifIndex/;" ifconfig.bat' + - '& C:/tools/cygwin/bin/sed.exe -i "s/kill -f/kill -W/;" conf.sh stop.pl' + - '& cmd.exe /C ifconfig.bat up; ""' + - 'Start-Sleep 2' + - '$Env:Path = "C:/tools/cygwin/bin;$Env:Path"' + - '& sh.exe runall.sh $TEST_PARALLEL_JOBS' + - 'If (Test-Path C:/CrashDumps/*) { dir C:/CrashDumps; Throw }' + artifacts: + untracked: true + expire_in: "1 week" + when: on_failure + only: + - schedules + - tags + - web + .unit_test: &unit_test_job <<: *default_triggering_rules stage: unit @@ -985,33 +1009,12 @@ msvc:windows:amd64: VSCONF: Release system:msvc:windows:amd64: - stage: system - tags: - - windows - - amd64 + <<: *windows_system_test_job variables: VSCONF: Release - script: - - 'Push-Location bin/tests/system' - - '$ifIndex = Get-NetIPInterface -AddressFamily IPv4 -InterfaceMetric 75 | Select-Object -ExpandProperty ifIndex' - - '& C:/tools/cygwin/bin/sed.exe -i "s/^exit.*/netsh interface ipv4 set dnsservers $ifIndex dhcp/; s/\(name\|interface\)=Loopback/$ifIndex/;" ifconfig.bat' - - '& C:/tools/cygwin/bin/sed.exe -i "s/kill -f/kill -W/;" conf.sh stop.pl' - - '& cmd.exe /C ifconfig.bat up; ""' - - 'Start-Sleep 2' - - '$Env:Path = "C:/tools/cygwin/bin;$Env:Path"' - - '& sh.exe runall.sh $TEST_PARALLEL_JOBS' - - 'If (Test-Path C:/CrashDumps/*) { dir C:/CrashDumps; Throw }' dependencies: - msvc:windows:amd64 needs: ["msvc:windows:amd64"] - artifacts: - untracked: true - expire_in: "1 week" - when: on_failure - only: - - schedules - - tags - - web msvc-debug:windows:amd64: <<: *windows_build_job @@ -1021,3 +1024,11 @@ msvc-debug:windows:amd64: - schedules - tags - web + +system:msvc-debug:windows:amd64: + <<: *windows_system_test_job + variables: + VSCONF: Debug + dependencies: + - msvc-debug:windows:amd64 + needs: ["msvc-debug:windows:amd64"] From 5a4a6b5e91c318c5f5bf655572de77f3a37bcdf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Wed, 11 Dec 2019 12:04:29 +0100 Subject: [PATCH 6/7] Add a job creating a release tarball to GitLab CI Add a GitLab CI job (which is run only if all other jobs in a pipeline succeed) that builds a BIND release tarball, i.e. fetches the source tarball from the tarball building job, creates Windows zips, puts certain parts of BIND documentation into the appropriate places, and packs it all up into a single tarball whose contents can be subsequently signed and published. --- .gitlab-ci.yml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b947489952..b8c1e1784a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,6 +32,7 @@ stages: - docs - push - postcheck + - release ### Runner Tag Templates @@ -1032,3 +1033,45 @@ system:msvc-debug:windows:amd64: dependencies: - msvc-debug:windows:amd64 needs: ["msvc-debug:windows:amd64"] + +# Job producing a release tarball + +release:sid:amd64: + <<: *debian_sid_amd64_image + stage: release + script: + # Determine BIND version + - source version + - export BIND_DIRECTORY="bind-${MAJORVER}.${MINORVER}.${PATCHVER}${RELEASETYPE}${RELEASEVER}" + # Remove redundant files and system test utilities from Windows build artifacts + - find Build/Release/ -name "*.pdb" -print -delete + - find Build/Debug/ \( -name "*.bsc" -o -name "*.idb" \) -print -delete + - find Build/ -regextype posix-extended -regex "Build/.*/($(find bin/tests/ -type f | sed -nE "s|^bin/tests(/system)?/win32/(.*)\.vcxproj$|\2|p" | paste -d"|" -s))\..*" -print -delete + # Create Windows zips + - openssl dgst -sha256 "${BIND_DIRECTORY}.tar.${TARBALL_EXTENSION}" | tee Build/Release/SHA256 Build/Debug/SHA256 + - ( cd Build/Release; zip "../../BIND${BIND_DIRECTORY#bind-}.x64.zip" * ) + - ( cd Build/Debug; zip "../../BIND${BIND_DIRECTORY#bind-}.debug.x64.zip" * ) + # Prepare release tarball contents (tarballs + zips + documentation) + - mkdir -p release/doc/arm + - pushd release + - mv "../${BIND_DIRECTORY}.tar.${TARBALL_EXTENSION}" ../BIND*.zip . + - tar --extract --file="${BIND_DIRECTORY}.tar.${TARBALL_EXTENSION}" + - mv "${BIND_DIRECTORY}"/{CHANGES*,COPYRIGHT,LICENSE,README,srcid} . + - mv "${BIND_DIRECTORY}"/doc/arm/{Bv9ARM{*.html,.pdf},man.*,notes.{html,pdf,txt}} doc/arm/ + - rm -rf "${BIND_DIRECTORY}" + - cp doc/arm/notes.html "RELEASE-NOTES-${BIND_DIRECTORY}.html" + - cp doc/arm/notes.pdf "RELEASE-NOTES-${BIND_DIRECTORY}.pdf" + - cp doc/arm/notes.txt "RELEASE-NOTES-${BIND_DIRECTORY}.txt" + - popd + # Create release tarball + - tar --create --file="${CI_COMMIT_TAG}.tar.gz" --gzip release/ + dependencies: + - tarball-create:sid:amd64 + - msvc:windows:amd64 + - msvc-debug:windows:amd64 + only: + - tags + artifacts: + paths: + - "*.tar.gz" + expire_in: "1 week" From 922a2ae44afb48c7033ee034f744a72f4549e308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Wed, 11 Dec 2019 12:04:29 +0100 Subject: [PATCH 7/7] Drop Jenkins mentions from release issue template Update the release issue template to account for the fact that Jenkins is no longer needed for creating BIND release tarballs. --- .gitlab/issue_templates/Release.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab/issue_templates/Release.md b/.gitlab/issue_templates/Release.md index 5ee3a6c01a..3bac8a2a0c 100644 --- a/.gitlab/issue_templates/Release.md +++ b/.gitlab/issue_templates/Release.md @@ -33,8 +33,7 @@ ## Before the ASN Deadline (for ASN Releases) or the Public Release Date (for Regular Releases) - - [ ] ***(QA)*** Run the `make release` Jenkins jobs to produce the tarballs and zips. - - [ ] ***(QA)*** Verify the results of `make release` Jenkins jobs and prepare a QA report for the releases to be published. + - [ ] ***(QA)*** Verify GitLab CI results for the tags created and prepare a QA report for the releases to be published. - [ ] ***(QA)*** Request signatures for the tarballs. - [ ] ***(Signers)*** Sign the tarballs. - [ ] ***(QA)*** Check tarball signatures.