From 262dfd024089757b17168c299d392b685f20059c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Sat, 30 May 2026 10:30:38 +0200 Subject: [PATCH] build_all: parallelize manual/pyinstaller/flatpak/appimage Serial prep (venv tool installs + flatpak runtimes + wheel, which the AppImage depends on), then the four heavy builds run concurrently. The shared venv is only written during prep, so the parallel builds (read-only on the venv) don't race on pip. Per-step logs under dist/.build-logs/; failing logs are printed. --serial falls back to one-at-a-time. Co-Authored-By: Claude Opus 4.8 --- build_all.sh | 178 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 123 insertions(+), 55 deletions(-) diff --git a/build_all.sh b/build_all.sh index 50465b5..b5bbe24 100755 --- a/build_all.sh +++ b/build_all.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Build every distribution channel of testium, in order: +# Build every distribution channel of testium: # 1. Manual PDF -> dist/testium-manual-.pdf # 2. Wheel -> dist/testium--py3-none-any.whl (PEP 427 name) # 3. PyInstaller binary -> dist/testium- @@ -11,6 +11,15 @@ # By default, a step is skipped if its artifact already exists in dist/. # Pass --clean to remove existing dist/ artifacts and rebuild everything. # +# Parallelism: the wheel is built first (the AppImage installs it), then the +# manual, PyInstaller, Flatpak and AppImage builds run concurrently. The shared +# venv at test/tmp/.venv is only WRITTEN during the serial prep phase (the +# `pip install` of build/sphinx/pyinstaller); the parallel builds only read it, +# so there is no concurrent-pip race. Pass --serial to build one step at a time +# (useful when debugging or on a resource-constrained machine). Per-step output +# of the parallel phase is captured under dist/.build-logs/.log and the +# log of any failing step is printed at the end. +# # All artifacts are collected (copied) under /dist/. Original outputs in # src/dist/, package/*/dist/, doc/manual/ are left in place. Wheel and AppImage # keep their original names (which already contain the version); manual, @@ -26,9 +35,11 @@ set -e CLEAN=0 +SERIAL=0 for arg in "$@"; do case "$arg" in --clean|-c) CLEAN=1 ;; + --serial) SERIAL=1 ;; *) echo "Unknown option: $arg" >&2; exit 1 ;; esac done @@ -70,50 +81,77 @@ step() { echo "================================================================" } -skip() { echo " (already built — skipping)"; } +# ---------- artifact paths ---------------------------------------------------- -# 1. Manual PDF MANUAL="$DIST_DIR/testium-manual-${VERSION}.pdf" -step "1/5 Manual PDF (version $VERSION)" -if [ ! -f "$MANUAL" ]; then - python -m pip install --quiet --upgrade sphinx linuxdoc - bash "$SCRIPT_DIR/doc/manual/sphinx/build_doc.sh" - cp -f "$SCRIPT_DIR/doc/manual/testium_manual.pdf" "$MANUAL" -else - skip -fi +PYI_BIN="$DIST_DIR/testium-${VERSION}" +FLATPAK_BUNDLE="$DIST_DIR/testium-${VERSION}.flatpak" +wheel_in_dist() { ls -1t "$DIST_DIR"/testium-${VERSION}-*.whl 2>/dev/null | head -1; } +appimage_in_dist() { ls -1t "$DIST_DIR"/Testium-${VERSION}-*.AppImage 2>/dev/null | head -1; } -# 2. Wheel — PEP 427 name kept (already contains version) -step "2/5 Wheel (version $VERSION)" -WHEEL=$(ls -1t "$DIST_DIR"/testium-${VERSION}-*.whl 2>/dev/null | head -1) -if [ -z "$WHEEL" ]; then - python -m pip install --quiet --upgrade build +# ---------- per-step build functions (assume tools are installed) ------------- + +build_wheel() { + if [ -n "$(wheel_in_dist)" ]; then echo "wheel: already built — skipping"; return 0; fi + echo "wheel: building" ( cd "$SCRIPT_DIR/src" rm -rf dist build *.egg-info python -m build --wheel ) - WHEEL_SRC=$(ls -1t "$SCRIPT_DIR/src/dist"/*.whl | head -1) - WHEEL="$DIST_DIR/$(basename "$WHEEL_SRC")" - cp -f "$WHEEL_SRC" "$WHEEL" -else - skip -fi + local src; src=$(ls -1t "$SCRIPT_DIR/src/dist"/*.whl | head -1) + cp -f "$src" "$DIST_DIR/$(basename "$src")" + echo "wheel: done" +} -# 3. PyInstaller binary -PYI_BIN="$DIST_DIR/testium-${VERSION}" -step "3/5 PyInstaller binary (version $VERSION)" -if [ ! -f "$PYI_BIN" ]; then - python -m pip install --quiet --upgrade pyinstaller +build_manual() { + if [ -f "$MANUAL" ]; then echo "manual: already built — skipping"; return 0; fi + echo "manual: building" + bash "$SCRIPT_DIR/doc/manual/sphinx/build_doc.sh" + cp -f "$SCRIPT_DIR/doc/manual/testium_manual.pdf" "$MANUAL" + echo "manual: done" +} + +build_pyinstaller() { + if [ -f "$PYI_BIN" ]; then echo "pyinstaller: already built — skipping"; return 0; fi + echo "pyinstaller: building" bash "$SCRIPT_DIR/package/pyinstaller/build.sh" cp -f "$SCRIPT_DIR/package/pyinstaller/dist/testium" "$PYI_BIN" -else - skip -fi + echo "pyinstaller: done" +} + +build_flatpak() { + if [ -f "$FLATPAK_BUNDLE" ]; then echo "flatpak: already built — skipping"; return 0; fi + echo "flatpak: building" + ( + cd "$SCRIPT_DIR/package/flatpak" + bash build.sh + ) + cp -f "$SCRIPT_DIR/package/flatpak/testium.flatpak" "$FLATPAK_BUNDLE" + echo "flatpak: done" +} + +build_appimage() { + if [ -n "$(appimage_in_dist)" ]; then echo "appimage: already built — skipping"; return 0; fi + echo "appimage: building" + ( + cd "$SCRIPT_DIR/package/appimage" + bash build.sh + ) + local src; src=$(ls -1t "$SCRIPT_DIR/package/appimage"/*.AppImage 2>/dev/null | head -1) + cp -f "$src" "$DIST_DIR/$(basename "$src")" + chmod +x "$DIST_DIR/$(basename "$src")" + echo "appimage: done" +} + +# ---------- serial prep: tool installs (shared venv) + flatpak runtimes ------- + +step "Prep: build tools + runtimes (serial — shared venv)" + +[ -f "$MANUAL" ] || python -m pip install --quiet --upgrade sphinx linuxdoc +[ -n "$(wheel_in_dist)" ] || python -m pip install --quiet --upgrade build +[ -f "$PYI_BIN" ] || python -m pip install --quiet --upgrade pyinstaller -# 4. Flatpak bundle -FLATPAK_BUNDLE="$DIST_DIR/testium-${VERSION}.flatpak" -step "4/5 Flatpak bundle (version $VERSION)" if [ ! -f "$FLATPAK_BUNDLE" ]; then FLATPAK_DEPS=( "org.kde.Platform//6.10" @@ -130,35 +168,65 @@ if [ ! -f "$FLATPAK_BUNDLE" ]; then flatpak install --user --noninteractive flathub "$dep" fi done - ( - cd "$SCRIPT_DIR/package/flatpak" - bash build.sh - ) - cp -f "$SCRIPT_DIR/package/flatpak/testium.flatpak" "$FLATPAK_BUNDLE" -else - skip fi -# 5. AppImage -step "5/5 AppImage (version $VERSION)" -APPIMAGE=$(ls -1t "$DIST_DIR"/Testium-${VERSION}-*.AppImage 2>/dev/null | head -1) -if [ -z "$APPIMAGE" ]; then - ( - cd "$SCRIPT_DIR/package/appimage" - bash build.sh - ) - APPIMAGE_SRC=$(ls -1t "$SCRIPT_DIR/package/appimage"/*.AppImage 2>/dev/null | head -1) - APPIMAGE="$DIST_DIR/$(basename "$APPIMAGE_SRC")" - cp -f "$APPIMAGE_SRC" "$APPIMAGE" - chmod +x "$APPIMAGE" +# ---------- serial: wheel (the AppImage installs it) -------------------------- + +step "1/5 Wheel (version $VERSION)" +build_wheel + +# ---------- build the rest -------------------------------------------------- + +REST=(manual pyinstaller flatpak appimage) + +if [ "$SERIAL" -eq 1 ]; then + n=2 + for name in "${REST[@]}"; do + step "$n/5 $name (version $VERSION)" + "build_$name" + n=$((n + 1)) + done else - skip + step "2-5/5 manual + pyinstaller + flatpak + appimage (parallel)" + LOGDIR="$DIST_DIR/.build-logs" + mkdir -p "$LOGDIR" + declare -A PID2NAME + for name in "${REST[@]}"; do + log="$LOGDIR/$name.log" + echo " -> launching $name (log: $log)" + ( "build_$name" ) >"$log" 2>&1 & + PID2NAME[$!]="$name" + done + + FAILED=() + for pid in "${!PID2NAME[@]}"; do + name="${PID2NAME[$pid]}" + if wait "$pid"; then + echo " -> $name: OK" + else + echo " -> $name: FAILED (rc=$?)" + FAILED+=("$name") + fi + done + + if [ "${#FAILED[@]}" -gt 0 ]; then + for name in "${FAILED[@]}"; do + echo + echo "===================== $name log =====================" + cat "$LOGDIR/$name.log" + done + echo >&2 + echo "BUILD FAILED: ${FAILED[*]} (logs under $LOGDIR)" >&2 + exit 1 + fi fi +# ---------- summary ----------------------------------------------------------- + step "All packages built" printf " manual : %s\n" "$MANUAL" -printf " wheel : %s\n" "$WHEEL" +printf " wheel : %s\n" "$(wheel_in_dist)" printf " pyinstaller : %s\n" "$PYI_BIN" printf " flatpak : %s\n" "$FLATPAK_BUNDLE" -printf " appimage : %s\n" "$APPIMAGE" +printf " appimage : %s\n" "$(appimage_in_dist)" printf " release_note : %s\n" "$RELEASE_NOTE"