diff --git a/scripts/bootstrap-pacman.sh b/scripts/bootstrap-pacman.sh index 6377ee8..e594054 100755 --- a/scripts/bootstrap-pacman.sh +++ b/scripts/bootstrap-pacman.sh @@ -475,3 +475,48 @@ main() { if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then main "$@" fi + +# Run a command, and kill it if it stops saying anything. +# +# watched +# +# Returns the command's status, or 2 if it was killed for silence. +# +# WHY IT EXISTS. python's build ran for ten hours. It was not slow: it was +# spinning in a shell loop waiting for an X server that this port will never +# have, printing nothing. Nothing distinguished it from a long compile except +# that the log had not grown -- and that is a measurement, so it can be made +# automatically. +# +# Both stages need it, and stage 2 needs it MORE: stage 2 runs the test suites +# that stage 1 skipped, and a test suite is the likeliest thing in a build to +# wait forever on a terminal, a network socket, or a display. +# +# Killed as a PROCESS GROUP -- set -m gives the subshell its own, so make and +# every compiler under it die too. Killing the pid alone leaves the tree +# running, holding the disk, invisible to the driver that thinks it stopped it. +watched() { + local log="$1"; shift + local limit="${EL_STALL_MIN:-45}" + if [ "$limit" -eq 0 ]; then + "$@" > "$log" 2>&1 + return $? + fi + set -m + ( "$@" ) > "$log" 2>&1 & + local pid=$! last=0 still=0 sz + set +m + while kill -0 "$pid" 2>/dev/null; do + sleep 60 + sz=$(stat -c %s "$log" 2>/dev/null || echo 0) + if [ "$sz" -eq "$last" ]; then still=$((still + 1)); else still=0; fi + last="$sz" + if [ "$still" -ge "$limit" ]; then + printf '\n== driver: no output for %s minutes, killing ==\n' "$limit" >> "$log" + kill -9 -- "-$pid" 2>/dev/null + wait "$pid" 2>/dev/null + return 2 + fi + done + wait "$pid" +} diff --git a/scripts/build-stage1.sh b/scripts/build-stage1.sh index 877ce92..3e8bf11 100755 --- a/scripts/build-stage1.sh +++ b/scripts/build-stage1.sh @@ -26,181 +26,9 @@ STATE="$WORK/stage1.state" # Build order. It follows link-time dependencies, not pacman metadata: # --nodeps means pacman never checks, so anything a compiler actually needs # must already exist. Within a group the order is free. -STAGE1_PACKAGES=( - # Foundation: headers, then the C library, then the compiler chain. - linux-api-headers glibc binutils gcc - # Compression and crypto, needed by libarchive and curl further down. - zlib bzip2 xz zstd lz4 openssl - # Terminal handling: bash links against readline, readline against ncurses. - ncurses readline - # The shell, and the coreutils prerequisites Arch declares. - attr acl gmp mpfr libcap bash coreutils - # Text and file tools the build systems themselves call. - sed grep gawk findutils diffutils file which patch - # Archivers, then the library pacman reads packages with. - tar gzip expat libarchive - # Build systems. - m4 autoconf automake libtool make pkgconf - # pacman's network and signature stack. - libnghttp2 libpsl curl libgpg-error libassuan gnupg gpgme - # System skeleton: without these a rootfs has no /etc/passwd, no zones, - # no /etc/services -- and nothing boots to a usable shell. - filesystem iana-etc tzdata licenses shadow util-linux - # Named by the chroot test, not guessed. Installing the repo into a - # rootfs and entering it turned "does it work?" into a precise list: - # - libcap needs pam; openssl needs brotli; libarchive needs libxml2 - # - pacman itself asks for systemd, pacman-mirrorlist and - # libmakepkg-dropins - # Sixty-eight successful builds proved none of this. One chroot did. - # - # The chroot also named libselinux, and libselinux is NOT on this line, - # because that reading of it was wrong. Arch has no libselinux package at - # all -- the clone 404s. What the chroot saw was a HOST artefact: Ubuntu - # carries libselinux1-dev, coreutils probes for selinux/selinux.h - # unconditionally, and ours came out linked to a library the target will - # never contain. The answer is --without-selinux per package, which is - # what Arch's own build chroot gets for free by not having the header. - pam brotli libxml2 systemd pacman-mirrorlist libmakepkg-dropins - # The closure, named by pacman's own resolver rather than guessed. - # - # Everything above was added because a BUILD stopped. These were added - # because scripts/test-chroot.sh ran `pacman -Syp` with --nodeps OFF -- - # the check the whole bootstrap skips -- and the resolver listed exactly - # what the repository still owes. Nothing here is speculative. - # - # It is the MINIMAL closure, and two measurements shaped it. Dropping the - # python-brotli sub-package took the list from 70 unresolved names to 47 - # and removed `python` outright, with libffi, mpdecimal and gdbm behind - # it. Dropping systemd-ukify and systemd-tests removed five more python - # packages, and ukify cannot run on s390x at all. Both are recorded in - # TODO.md rather than left implicit. - # - # An audit of the remaining names against the ARTEFACTS found two that - # were declared and never linked: guile by make, and libisl.so by gcc. - # Both get their declaration removed, because it should describe the - # binary we shipped. - # - # Building isl instead was the first plan, and it is recorded here because - # the reason it failed is the kind that wastes an afternoon: the Arch - # packaging repo for isl was last touched in 2017 and its only source URL - # is isl.gforge.inria.fr, which died with INRIA's GForge. There is nothing - # to build. That flipped the decision -- not a change of mind, new - # evidence. - # - # These will pull their own dependencies. That is expected: the resolver - # will name the next round as precisely as it named this one. - audit ca-certificates cryptsetup dbus elfutils gettext gnutls hwdata - icu jansson kbd kmod krb5 libcap-ng libgcrypt libidn2 libksba - libmpc libnsl libseccomp libssh2 libtirpc libunistring libusb libxcrypt - nettle npth openldap pambase pcre2 perl pinentry sqlite tpm2-tss - # Closure, second round. The first thirty-five pulled these in, exactly as - # the comment above predicted, and the resolver named them just as - # precisely. - # - # THE MEASUREMENT THAT MATTERS HERE IS THE ONE THAT CHANGED ITS ANSWER. - # Four of these were going to be avoided by dropping a sub-package -- - # sqlite-tcl, sqlite-analyzer, the openldap server, debuginfod -- on the - # reasoning that had removed python-brotli earlier. Measured instead of - # assumed, tcl, unixodbc and libmicrohttpd each cost ZERO new packages: - # the closure had filled in around them. Building them is cheaper than - # four hooks, and it does not leave sqlite shipping an sqltclsh that - # cannot start. - # - # python still costs three (libffi, mpdecimal, gdbm) and is still avoided - # -- but for the ABI reason, not the cost one: python-audit and - # python-capng are cp313 wheels and Arch ships 3.14. Their hooks say so. - # - db5.3 gdbm e2fsprogs gnulib-l10n json-c keyutils p11-kit libsasl - libsodium libtasn1 libverto lmdb popt lvm2 tcl unixodbc libmicrohttpd - # ca-certificates-mozilla, which is the only thing here that is not a - # library. It is the LIST OF ROOT CERTIFICATES -- ca-certificates itself - # is only the trust machinery around it, so without this the target has a - # trust store containing nothing, and every HTTPS verification fails. - # curl declares ca-certificates, and pacman fetches through curl. - # - # It has no packaging repo of its own: the clone 404s, which - # gitlab.archlinux.org reports by asking for a login -- the same - # misleading shape that made libselinux look like a network problem. nss - # produces it as a sub-package, so nss is what gets built, and nspr comes - # with it. - # - # Measured before committing to it rather than estimated: nspr costs - # nothing new, nss needs only nspr plus mercurial on the host, and - # hg.mozilla.org answers from this build host (HTTP 302 in 0.3s -- worth - # checking, since dev.gnupg.org does not answer at all and libassuan - # needed a source change because of it). - nspr nss - # Closure, third round, and it is two packages. Both were pulled in by - # what the second round added, and both cost nothing further: libffi by - # libp11-kit, libevent by libverto. - # - # They are worth a line because of what they unblocked. p11-kit builds - # into three packages, and libp11-kit's dependency on libffi was holding - # up the whole chain ca-certificates -> ca-certificates-utils -> p11-kit - # -> libp11-kit. The resolver reported it as "ca-certificates required by - # curl" -- four links away from the missing name, and nothing in that - # message points at libffi. - libffi libevent - # Closure, fourth round -- and it closed to NOTHING, which is the point - # worth recording. It asked for libaio and thin-provisioning-tools, both - # wanted by lvm2 alone. thin-provisioning-tools is Rust now, so that was a - # language runtime arriving through a fourth-order dependency. - # - # Nothing in the repository wants `lvm2`. Checked across every .PKGINFO: - # cryptsetup wants device-mapper, and device-mapper is the OTHER package - # this same source produces. Dropping the lvm2 sub-package removed both - # entries and the Rust question with them -- see patches/pkgbuild/lvm2.sh. - # - # 35 packages, then 19, then 2, then 0. The closure has to be recomputed - # after each round rather than once: lvm2 DECLARED libaio all along, and - # the third round could not see it because lvm2 had not been built yet. - # What STAGE 2 needs in order to exist, which is a different question from - # what the repository needs to resolve. - # - # Stage 2 rebuilds everything INSIDE the stage-1 rootfs, so the tools that - # do the rebuilding have to be in that rootfs. The resolver never asked - # for these -- nothing in the repository depends on them -- so the closure - # rounds could not surface them. Enumerated instead from what makepkg and - # an autotools build actually invoke. - # - # fakeroot is the one that decides whether stage 2 can start at all: - # makepkg runs package() under it, and refuses to run as root. bison, - # flex, texinfo and groff are what the sources themselves call -- gcc, - # glibc and binutils all want makeinfo, and a great many configure scripts - # want bison. - # - # sudo is deliberately NOT here. makepkg needs it only for --syncdeps, and - # stage 2 passes --nodeps, so it would be a setuid binary in the chroot - # for no reason. - fakeroot bison flex texinfo groff - # git, and it is not optional: 51 of the 159 PKGBUILDs take their sources - # from a git+https URL, and makepkg re-validates that clone even with - # --noextract. Without git in the chroot, stage 2 can rebuild a third of - # the repository and no more. - # - # Its own three: perl-error, perl-mailtools (which brings perl-timedate) - # and zlib-ng. Measured, not guessed -- and read from the depends array - # rather than from my own tool, which had reported `zsh` as a dependency - # of git. It is not; the tool's regex was catching a neighbouring array. - perl-error perl-timedate perl-mailtools zlib-ng git - # meson and cmake, which is where python re-enters -- and the distinction - # matters, because dropping python earlier was not a mistake. - # - # At stage 1 the question was what the REPOSITORY must supply: python was - # wanted only by python-brotli and python-libseccomp, wheels that Arch's - # own python could not load anyway, so they went and python went with them. - # Here the question is what the BUILD ENVIRONMENT must contain, and meson - # is written in Python. Ten of the 159 PKGBUILDs call arch-meson; four call - # cmake. Different question, different answer. - # - # Measured: mpdecimal, python, ninja, python-tqdm and meson, then cmake - # with cppdap, jsoncpp, libuv, rhash and hicolor-icon-theme behind it. - # Nothing further. - mpdecimal python ninja python-tqdm meson - cppdap jsoncpp libuv rhash hicolor-icon-theme cmake - # And finally the package manager itself, built as an Arch package. - pacman -) +# The list and its order live in one file, read by both stages -- see the +# header of packages.sh for why stage 2 must not derive its own. +source "$HERE/packages.sh" # Kill a build that has stopped producing output. # @@ -235,31 +63,8 @@ STAGE1_PACKAGES=( # re-declared build_package into a fresh shell, which loses $WORK, $REPO, # $PATCH_DIR and every other function it calls -- a guard that would have # broken the thing it was guarding. -build_watched() { - local p="$1" log="$2" - local limit="${EL_STALL_MIN:-45}" - if [ "$limit" -eq 0 ]; then - build_package "$p" > "$log" 2>&1 - return $? - fi - set -m - ( build_package "$p" ) > "$log" 2>&1 & - local pid=$! last=0 still=0 sz - set +m - while kill -0 "$pid" 2>/dev/null; do - sleep 60 - sz=$(stat -c %s "$log" 2>/dev/null || echo 0) - if [ "$sz" -eq "$last" ]; then still=$((still + 1)); else still=0; fi - last="$sz" - if [ "$still" -ge "$limit" ]; then - printf '\n== driver: no output for %s minutes, killing ==\n' "$limit" >> "$log" - kill -9 -- "-$pid" 2>/dev/null - wait "$pid" 2>/dev/null - return 2 - fi - done - wait "$pid" -} +# The stall guard moved to bootstrap-pacman.sh when stage 2 needed it too. +build_watched() { watched "$2" build_package "$1"; } built() { grep -qxF "$1" "$STATE" 2>/dev/null; } # Appended only once: a forced rebuild of an already-built package must not diff --git a/scripts/build-stage2.sh b/scripts/build-stage2.sh index c324c0c..b14e1a1 100755 --- a/scripts/build-stage2.sh +++ b/scripts/build-stage2.sh @@ -27,6 +27,9 @@ set -uo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Stage 2 rebuilds the same packages in the same order; packages.sh explains +# why that order is not re-derived here. +source "$HERE/packages.sh" WORK="${WORK:-$HOME/work/arch-s390x}" REPO1="${REPO1:-$WORK/repo/s390x}" REPO2="${REPO2:-$WORK/repo2/s390x}" @@ -78,12 +81,24 @@ CHROOT_PKGS=( log() { printf '\n== %s ==\n' "$*"; } die() { printf 'stage2: %s\n' "$*" >&2; exit 1; } -require_space() { +# A PREDICATE and a fatal check, kept apart on purpose. +# +# require_space calls die, which exits. Using it inside the rebuild loop as +# `require_space || break` looks like a clean early stop and is not one: the +# break is unreachable, the run ends mid-loop, and the summary naming which +# packages were rebuilt and which failed is never printed. At the start of the +# run, exiting IS the right answer -- there is nothing to summarise yet. +space_ok() { local free_mb free_mb=$(df -Pm "$WORK" | awk 'NR==2 {print $4}') - [ "${free_mb:-0}" -ge 8192 ] || die "only ${free_mb} MiB free under $WORK; need 8192" + if [ "${free_mb:-0}" -lt 8192 ]; then + printf ' only %s MiB free under %s; need 8192\n' "${free_mb:-0}" "$WORK" >&2 + return 1 + fi } +require_space() { space_ok || die "not enough disk space to start"; } + make_rootfs() { log "Populating the stage-2 rootfs from stage 1" cat > "$CONF" < "$WORK/stage2-install.txt" 2>&1 \ || { tail -20 "$WORK/stage2-install.txt" >&2; die "populate failed"; } printf ' %s packages installed\n' "$(sudo ls "$ROOT/var/lib/pacman/local" | wc -l)" + + # Put stage 2's own output back on top of it. + # + # make_rootfs wipes and repopulates from stage 1 on EVERY run, which is + # what makes the chroot reproducible -- and what would make stage 2 + # unresumable, because stage2.state survives while the packages it names do + # not. The second invocation would say "already rebuilt, skipping" about + # packages that had just been thrown away, and the next build would link + # against stage-1 libraries while the record claimed otherwise. Silent, and + # the kind of thing found weeks later in an artefact. + # + # Stage 2 is a hundred and thirty-three packages. It will not finish in one + # invocation, so resuming has to be correct rather than approximately + # correct. + # + # --nodeps for the same reason the per-package install uses it: a chroot + # halfway through stage 2 is a mixed population, some packages declaring + # versioned soname dependencies and some declaring names. -U is fed the + # files directly, so --nodeps installs exactly these and not a resolved + # closure -- correct here, since stage 1 already supplied the closure just + # above. + shopt -s nullglob + local back=("$REPO2"/*.pkg.tar.*) + shopt -u nullglob + if [ "${#back[@]}" -gt 0 ]; then + sudo pacman --root "$ROOT" --config "$CONF" --cachedir "$CACHE" \ + --noconfirm --nodeps -U "${back[@]}" \ + > "$WORK/stage2-restore.txt" 2>&1 \ + || { tail -20 "$WORK/stage2-restore.txt" >&2; die "restoring stage-2 output failed"; } + printf ' %s stage-2 package(s) restored\n' "${#back[@]}" + fi } configure_chroot() { @@ -395,6 +441,21 @@ stage2_build() { # the host's libraries and their verdict said nothing about the port. Here # they test what was actually built, which is the whole point of stage 2. local mkflags="--nodeps --ignorearch --skippgpcheck --skipchecksums" + # EL_NOCHECK=1 for the FIRST pass over the list, and only that. + # + # Stage 1 skipped every test suite because it ran against the host's + # libraries, so its verdict said nothing about the port. In here a suite + # tests what was actually built, which is worth having -- but not on the + # pass whose job is to find out whether a hundred and thirty-three packages + # can be rebuilt at all. glibc's suite alone is longer than most of the + # builds around it, and a suite is the likeliest place in a build to wait + # forever on a tty or a socket. + # + # So: one pass to get a complete stage-2 repository, then the suites, then + # stage 3 -- where they run on a self-hosted toolchain and their verdict is + # about the port rather than about the bootstrap. Off by default is wrong + # here; this must be asked for. + [ "${EL_NOCHECK:-0}" = "1" ] && mkflags="$mkflags --nocheck" if host_extract "$name"; then echo " extracting on the host (chroot bsdtar not usable yet)" ( cd "$dir" && LC_ALL=C.UTF-8 makepkg $mkflags -o -C -f ) || return 1 @@ -443,6 +504,23 @@ stage2_build() { --noconfirm --nodeps -U "${produced[@]}" > "$WORK/stage2-inst-$name.txt" 2>&1 || { tail -10 "$WORK/stage2-inst-$name.txt" >&2; return 1; } printf ' installed %s package(s)\n' "${#produced[@]}" + + # Clean up, but only now, and only because it worked. + # + # A hundred and thirty-three source trees plus their pkg/ staging do not fit + # on this disk. Filling it is not a hypothetical here: it happened once, and + # what it looked like was not "no space" -- it was I/O errors from unrelated + # virtual machines on the same host. Cheap to prevent, expensive to explain. + # + # AFTER the install, so nothing is thrown away until the package is proven + # to exist and to install. NOT on failure -- src/ and pkg/ are the whole + # evidence of what went wrong, and a build that failed is exactly the one + # worth looking at. makepkg -c would delete them either way. + # + # Only makepkg's own two directories, and only inside this package's + # checkout. The git tree, the PKGBUILD, the downloaded sources and the built + # package are all left alone. + ( cd "$dir" && rm -rf src pkg ) || true } # A pacman.conf that sees BOTH repositories: stage 2's output first, so a @@ -462,17 +540,29 @@ EOF rebuild() { write_conf2 mkdir -p "$REPO2" - local ok=0 fail=0 failed=() + local ok=0 fail=0 rc=0 failed=() for p in "$@"; do if grep -qxF "$p" "$STATE2" 2>/dev/null; then echo "== $p already rebuilt, skipping =="; continue fi + # Per package, not once at the start. The run is long enough that the + # disk state at the end has nothing to do with the disk state when it + # was checked, and the failure mode is not local to this script. + space_ok || { echo "== stage 2: stopping, disk too low =="; break; } log "stage 2: $p" - if stage2_build "$p" > "$WORK/stage2-log-$p.txt" 2>&1; then + # watched, not a plain call: see bootstrap-pacman.sh. A hundred and + # thirty-three packages is far too many to sit in front of, and one + # silent build would hold the whole run. + if watched "$WORK/stage2-log-$p.txt" stage2_build "$p"; then echo "$p" >> "$STATE2"; ok=$((ok + 1)); echo "OK $p" else + rc=$? fail=$((fail + 1)); failed+=("$p") - echo "FAIL $p (see $WORK/stage2-log-$p.txt)" + if [ "$rc" -eq 2 ]; then + echo "STALL $p (no output for ${EL_STALL_MIN:-45} min, killed)" + else + echo "FAIL $p (see $WORK/stage2-log-$p.txt)" + fi tail -5 "$WORK/stage2-log-$p.txt" | sed 's/^/ /' fi done @@ -495,10 +585,18 @@ main() { echo " sudo chroot --userspec=$BUILD_UID:$BUILD_GID $ROOT /usr/bin/bash -l" echo " or rebuild packages:" echo " bash $0 texinfo perl m4 autoconf ..." + echo " bash $0 --all # all ${#STAGE1_PACKAGES[@]}, in order" return 0 fi touch "$STATE2" - rebuild "$@" + # --all: the shared list, in its order. Typing a hundred and thirty-three + # names is not a workflow, and typing a subset of them is how an ordering + # gets quietly reinvented. + if [ "$1" = "--all" ]; then + rebuild "${STAGE1_PACKAGES[@]}" + else + rebuild "$@" + fi } main "$@" diff --git a/scripts/packages.sh b/scripts/packages.sh new file mode 100644 index 0000000..196562f --- /dev/null +++ b/scripts/packages.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +# The package list, and the ORDER, shared by both stages. +# +# It lived in build-stage1.sh until stage 2 needed it. Stage 2 rebuilds the +# same packages inside the chroot, and the order it needs is not a new +# question: this one is the actual dependency closure, arrived at over four +# rounds of resolver output (35 unsatisfied, then 19, then 2, then none) and +# then confirmed by a hundred and ninety builds that each found their +# dependencies already present. Deriving a second order for stage 2 would be +# re-deriving a fact this file already holds -- and getting it subtly wrong +# would show up as a build failure attributed to the package rather than to +# the ordering. +# +# Sourced, never executed. Both stages read STAGE1_PACKAGES from here. + +STAGE1_PACKAGES=( + # Foundation: headers, then the C library, then the compiler chain. + linux-api-headers glibc binutils gcc + # Compression and crypto, needed by libarchive and curl further down. + zlib bzip2 xz zstd lz4 openssl + # Terminal handling: bash links against readline, readline against ncurses. + ncurses readline + # The shell, and the coreutils prerequisites Arch declares. + attr acl gmp mpfr libcap bash coreutils + # Text and file tools the build systems themselves call. + sed grep gawk findutils diffutils file which patch + # Archivers, then the library pacman reads packages with. + tar gzip expat libarchive + # Build systems. + m4 autoconf automake libtool make pkgconf + # pacman's network and signature stack. + libnghttp2 libpsl curl libgpg-error libassuan gnupg gpgme + # System skeleton: without these a rootfs has no /etc/passwd, no zones, + # no /etc/services -- and nothing boots to a usable shell. + filesystem iana-etc tzdata licenses shadow util-linux + # Named by the chroot test, not guessed. Installing the repo into a + # rootfs and entering it turned "does it work?" into a precise list: + # - libcap needs pam; openssl needs brotli; libarchive needs libxml2 + # - pacman itself asks for systemd, pacman-mirrorlist and + # libmakepkg-dropins + # Sixty-eight successful builds proved none of this. One chroot did. + # + # The chroot also named libselinux, and libselinux is NOT on this line, + # because that reading of it was wrong. Arch has no libselinux package at + # all -- the clone 404s. What the chroot saw was a HOST artefact: Ubuntu + # carries libselinux1-dev, coreutils probes for selinux/selinux.h + # unconditionally, and ours came out linked to a library the target will + # never contain. The answer is --without-selinux per package, which is + # what Arch's own build chroot gets for free by not having the header. + pam brotli libxml2 systemd pacman-mirrorlist libmakepkg-dropins + # The closure, named by pacman's own resolver rather than guessed. + # + # Everything above was added because a BUILD stopped. These were added + # because scripts/test-chroot.sh ran `pacman -Syp` with --nodeps OFF -- + # the check the whole bootstrap skips -- and the resolver listed exactly + # what the repository still owes. Nothing here is speculative. + # + # It is the MINIMAL closure, and two measurements shaped it. Dropping the + # python-brotli sub-package took the list from 70 unresolved names to 47 + # and removed `python` outright, with libffi, mpdecimal and gdbm behind + # it. Dropping systemd-ukify and systemd-tests removed five more python + # packages, and ukify cannot run on s390x at all. Both are recorded in + # TODO.md rather than left implicit. + # + # An audit of the remaining names against the ARTEFACTS found two that + # were declared and never linked: guile by make, and libisl.so by gcc. + # Both get their declaration removed, because it should describe the + # binary we shipped. + # + # Building isl instead was the first plan, and it is recorded here because + # the reason it failed is the kind that wastes an afternoon: the Arch + # packaging repo for isl was last touched in 2017 and its only source URL + # is isl.gforge.inria.fr, which died with INRIA's GForge. There is nothing + # to build. That flipped the decision -- not a change of mind, new + # evidence. + # + # These will pull their own dependencies. That is expected: the resolver + # will name the next round as precisely as it named this one. + audit ca-certificates cryptsetup dbus elfutils gettext gnutls hwdata + icu jansson kbd kmod krb5 libcap-ng libgcrypt libidn2 libksba + libmpc libnsl libseccomp libssh2 libtirpc libunistring libusb libxcrypt + nettle npth openldap pambase pcre2 perl pinentry sqlite tpm2-tss + # Closure, second round. The first thirty-five pulled these in, exactly as + # the comment above predicted, and the resolver named them just as + # precisely. + # + # THE MEASUREMENT THAT MATTERS HERE IS THE ONE THAT CHANGED ITS ANSWER. + # Four of these were going to be avoided by dropping a sub-package -- + # sqlite-tcl, sqlite-analyzer, the openldap server, debuginfod -- on the + # reasoning that had removed python-brotli earlier. Measured instead of + # assumed, tcl, unixodbc and libmicrohttpd each cost ZERO new packages: + # the closure had filled in around them. Building them is cheaper than + # four hooks, and it does not leave sqlite shipping an sqltclsh that + # cannot start. + # + # python still costs three (libffi, mpdecimal, gdbm) and is still avoided + # -- but for the ABI reason, not the cost one: python-audit and + # python-capng are cp313 wheels and Arch ships 3.14. Their hooks say so. + # + db5.3 gdbm e2fsprogs gnulib-l10n json-c keyutils p11-kit libsasl + libsodium libtasn1 libverto lmdb popt lvm2 tcl unixodbc libmicrohttpd + # ca-certificates-mozilla, which is the only thing here that is not a + # library. It is the LIST OF ROOT CERTIFICATES -- ca-certificates itself + # is only the trust machinery around it, so without this the target has a + # trust store containing nothing, and every HTTPS verification fails. + # curl declares ca-certificates, and pacman fetches through curl. + # + # It has no packaging repo of its own: the clone 404s, which + # gitlab.archlinux.org reports by asking for a login -- the same + # misleading shape that made libselinux look like a network problem. nss + # produces it as a sub-package, so nss is what gets built, and nspr comes + # with it. + # + # Measured before committing to it rather than estimated: nspr costs + # nothing new, nss needs only nspr plus mercurial on the host, and + # hg.mozilla.org answers from this build host (HTTP 302 in 0.3s -- worth + # checking, since dev.gnupg.org does not answer at all and libassuan + # needed a source change because of it). + nspr nss + # Closure, third round, and it is two packages. Both were pulled in by + # what the second round added, and both cost nothing further: libffi by + # libp11-kit, libevent by libverto. + # + # They are worth a line because of what they unblocked. p11-kit builds + # into three packages, and libp11-kit's dependency on libffi was holding + # up the whole chain ca-certificates -> ca-certificates-utils -> p11-kit + # -> libp11-kit. The resolver reported it as "ca-certificates required by + # curl" -- four links away from the missing name, and nothing in that + # message points at libffi. + libffi libevent + # Closure, fourth round -- and it closed to NOTHING, which is the point + # worth recording. It asked for libaio and thin-provisioning-tools, both + # wanted by lvm2 alone. thin-provisioning-tools is Rust now, so that was a + # language runtime arriving through a fourth-order dependency. + # + # Nothing in the repository wants `lvm2`. Checked across every .PKGINFO: + # cryptsetup wants device-mapper, and device-mapper is the OTHER package + # this same source produces. Dropping the lvm2 sub-package removed both + # entries and the Rust question with them -- see patches/pkgbuild/lvm2.sh. + # + # 35 packages, then 19, then 2, then 0. The closure has to be recomputed + # after each round rather than once: lvm2 DECLARED libaio all along, and + # the third round could not see it because lvm2 had not been built yet. + # What STAGE 2 needs in order to exist, which is a different question from + # what the repository needs to resolve. + # + # Stage 2 rebuilds everything INSIDE the stage-1 rootfs, so the tools that + # do the rebuilding have to be in that rootfs. The resolver never asked + # for these -- nothing in the repository depends on them -- so the closure + # rounds could not surface them. Enumerated instead from what makepkg and + # an autotools build actually invoke. + # + # fakeroot is the one that decides whether stage 2 can start at all: + # makepkg runs package() under it, and refuses to run as root. bison, + # flex, texinfo and groff are what the sources themselves call -- gcc, + # glibc and binutils all want makeinfo, and a great many configure scripts + # want bison. + # + # sudo is deliberately NOT here. makepkg needs it only for --syncdeps, and + # stage 2 passes --nodeps, so it would be a setuid binary in the chroot + # for no reason. + fakeroot bison flex texinfo groff + # git, and it is not optional: 51 of the 159 PKGBUILDs take their sources + # from a git+https URL, and makepkg re-validates that clone even with + # --noextract. Without git in the chroot, stage 2 can rebuild a third of + # the repository and no more. + # + # Its own three: perl-error, perl-mailtools (which brings perl-timedate) + # and zlib-ng. Measured, not guessed -- and read from the depends array + # rather than from my own tool, which had reported `zsh` as a dependency + # of git. It is not; the tool's regex was catching a neighbouring array. + perl-error perl-timedate perl-mailtools zlib-ng git + # meson and cmake, which is where python re-enters -- and the distinction + # matters, because dropping python earlier was not a mistake. + # + # At stage 1 the question was what the REPOSITORY must supply: python was + # wanted only by python-brotli and python-libseccomp, wheels that Arch's + # own python could not load anyway, so they went and python went with them. + # Here the question is what the BUILD ENVIRONMENT must contain, and meson + # is written in Python. Ten of the 159 PKGBUILDs call arch-meson; four call + # cmake. Different question, different answer. + # + # Measured: mpdecimal, python, ninja, python-tqdm and meson, then cmake + # with cppdap, jsoncpp, libuv, rhash and hicolor-icon-theme behind it. + # Nothing further. + mpdecimal python ninja python-tqdm meson + cppdap jsoncpp libuv rhash hicolor-icon-theme cmake + # And finally the package manager itself, built as an Arch package. + pacman +)