From 92c5850e530ebdce0f9b536ef2f5fcf1c813b5e0 Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Wed, 19 Aug 2026 19:53:46 -0400 Subject: [PATCH] [FIX] driver: kill a build that has stopped producing output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit python's PKGBUILD hunts for a free X display before running make: export servernum=99 while ! xvfb-run -a -n "$servernum" /bin/true 2>/dev/null; do servernum=$((servernum+1)); done xvfb-run is not installed here. It exits 127, `!` inverts that to true, and the loop tries the next number. There is no exit condition for "the command does not exist", and the 2>/dev/null swallows the one line that would have said so. Measured before anyone noticed: ten hours of wall clock, four million PIDs, and a log frozen at `configure: creating Makefile`. Every signal said the build was healthy -- alive, 40% CPU, sitting in a plausible source directory. A process list cannot tell work from spinning. A stalled log next to a live makepkg can, and that is all this guard does. Forty-five minutes of COMPLETE silence, because real builds do go quiet: a long link, a test suite that prints nothing, gcc between bootstrap stages. The case this was written for ran two hundred times longer. EL_STALL_MIN=0 disables it. Tested against a real spin loop: detected, whole process tree killed, rc=2 so STALL reads differently from FAIL in the summary. --- FR --- Le PKGBUILD de python cherche un numéro d'affichage X libre avant make : export servernum=99 while ! xvfb-run -a -n "$servernum" /bin/true 2>/dev/null; do servernum=$((servernum+1)); done xvfb-run n'est pas installé ici. Il sort en 127, le `!` inverse, et la boucle essaie le numéro suivant. Il n'existe aucune condition de sortie pour « la commande n'existe pas », et le 2>/dev/null avale la seule ligne qui l'aurait dit. Mesuré avant que quiconque s'en aperçoive : dix heures d'horloge, quatre millions de PID, un journal figé à `configure: creating Makefile`. Tous les signaux disaient que le build allait bien — vivant, 40 % de CPU, dans un répertoire source plausible. Une liste de processus ne distingue pas le travail du sur-place. Un journal figé à côté d'un makepkg vivant, si — et c'est tout ce que fait ce garde-fou. Quarante-cinq minutes de silence COMPLET, car de vrais builds se taisent : une longue édition de liens, une suite de tests muette, gcc entre deux étages. Le cas qui l'a motivé a tourné deux cents fois plus longtemps. EL_STALL_MIN=0 le désactive. Éprouvé contre une vraie boucle infinie : détecté, arbre de processus tué en entier, rc=2 pour que STALL se lise autrement que FAIL dans le bilan. Assisted-by: Claude Opus 5 --- scripts/build-stage1.sh | 85 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 3 deletions(-) diff --git a/scripts/build-stage1.sh b/scripts/build-stage1.sh index 3d64da0..123e34e 100755 --- a/scripts/build-stage1.sh +++ b/scripts/build-stage1.sh @@ -183,10 +183,84 @@ STAGE1_PACKAGES=( # rather than from my own tool, which had reported `zsh` as a dependency # of git. It is not; the tool's regex was catching a neighbouring array. perl-error perl-timedate perl-mailtools zlib-ng git + # meson and cmake, which is where python re-enters -- and the distinction + # matters, because dropping python earlier was not a mistake. + # + # At stage 1 the question was what the REPOSITORY must supply: python was + # wanted only by python-brotli and python-libseccomp, wheels that Arch's + # own python could not load anyway, so they went and python went with them. + # Here the question is what the BUILD ENVIRONMENT must contain, and meson + # is written in Python. Ten of the 159 PKGBUILDs call arch-meson; four call + # cmake. Different question, different answer. + # + # Measured: mpdecimal, python, ninja, python-tqdm and meson, then cmake + # with cppdap, jsoncpp, libuv, rhash and hicolor-icon-theme behind it. + # Nothing further. + mpdecimal python ninja python-tqdm meson + cppdap jsoncpp libuv rhash hicolor-icon-theme cmake # And finally the package manager itself, built as an Arch package. pacman ) +# Kill a build that has stopped producing output. +# +# WHY THIS EXISTS. python's PKGBUILD contains +# +# while ! xvfb-run -a -n "$servernum" /bin/true 2>/dev/null; do +# servernum=$((servernum+1)); done +# +# and xvfb-run is not installed here. It exits 127, `!` inverts that to true, +# and the loop tries the next display number. Forever -- there is no exit +# condition for "the command does not exist", and the 2>/dev/null swallows the +# one line that would have said so. +# +# Measured before anyone noticed: TEN HOURS of wall clock, four million PIDs, +# and a log frozen at `configure: creating Makefile`. Every signal I had said +# the build was healthy -- the process was alive, burning 40% CPU, sitting in +# a plausible source directory. A process list cannot tell work from spinning. +# +# A STALLED LOG NEXT TO A LIVE makepkg CAN. That is the whole idea here. +# +# The threshold is deliberately generous. Real builds do go quiet: a long link, +# a test suite that prints nothing, gcc between bootstrap stages. Forty-five +# minutes of COMPLETE silence is not one of those, and the case this was +# written for ran two hundred times longer. EL_STALL_MIN=0 disables it. +# +# `set -m` rather than setsid or a bare `&`. The whole tree has to die, because +# makepkg forks children that outlive it -- killing the parent alone leaves +# them spinning, which is how ten hours happened. Job control gives the +# background job its own process group, so `kill -- -$pid` reaches all of it. +# +# A subshell, NOT `setsid bash -c "$(declare -f ...)"`. The first attempt here +# re-declared build_package into a fresh shell, which loses $WORK, $REPO, +# $PATCH_DIR and every other function it calls -- a guard that would have +# broken the thing it was guarding. +build_watched() { + local p="$1" log="$2" + local limit="${EL_STALL_MIN:-45}" + if [ "$limit" -eq 0 ]; then + build_package "$p" > "$log" 2>&1 + return $? + fi + set -m + ( build_package "$p" ) > "$log" 2>&1 & + local pid=$! last=0 still=0 sz + set +m + while kill -0 "$pid" 2>/dev/null; do + sleep 60 + sz=$(stat -c %s "$log" 2>/dev/null || echo 0) + if [ "$sz" -eq "$last" ]; then still=$((still + 1)); else still=0; fi + last="$sz" + if [ "$still" -ge "$limit" ]; then + printf '\n== driver: no output for %s minutes, killing ==\n' "$limit" >> "$log" + kill -9 -- "-$pid" 2>/dev/null + wait "$pid" 2>/dev/null + return 2 + fi + done + wait "$pid" +} + built() { grep -qxF "$1" "$STATE" 2>/dev/null; } mark() { echo "$1" >> "$STATE"; } @@ -197,7 +271,7 @@ main() { # reinstalls them silently builds with whatever was deployed weeks ago. # Cheap, idempotent, and it makes this repository the source of truth. install_host_shims - local ok=0 fail=0 failed=() + local ok=0 fail=0 rc=0 failed=() for p in "${STAGE1_PACKAGES[@]}"; do if built "$p"; then echo "== $p already built, skipping ==" @@ -205,12 +279,17 @@ main() { fi # A failure must not stop the run: one missing package should not hide # the state of the forty that follow. They are collected and reported. - if build_package "$p" > "$WORK/log-$p.txt" 2>&1; then + if build_watched "$p" "$WORK/log-$p.txt"; then mark "$p"; ok=$((ok + 1)) echo "OK $p" else + rc=$? fail=$((fail + 1)); failed+=("$p") - echo "FAIL $p (see $WORK/log-$p.txt)" + if [ "$rc" -eq 2 ]; then + echo "STALL $p (no output for ${EL_STALL_MIN:-45} min, killed)" + else + echo "FAIL $p (see $WORK/log-$p.txt)" + fi fi done echo