flatpak: run host interpreters via flatpak-spawn; validation --mode flag

py_func, lua_func and the run item now reach host binaries through
`flatpak-spawn --host` instead of trying to load them under the
sandbox runtime (which fails with a glibc ABI mismatch). Adds
`--talk-name=org.freedesktop.Flatpak` to the manifest, stages the
/app/lib/testium tree under /tmp so the host can read it, and drops
the dead `_FLATPAK_HOST_DIRS` / lib-injection code paths that the
new approach makes obsolete.

Validation suite gains a `--mode source|wheel|pyinstaller|flatpak|
appimage` flag so the same item set can run against every packaging
channel; per-mode report file names avoid clobbering.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

package/flatpak/org.testium.Testium.yaml

@@ -16,6 +16,11 @@ finish-args:
   - --filesystem=home
   - --filesystem=/tmp
   - --filesystem=host-os
+  # Allow flatpak-spawn --host to launch host binaries (Python, Lua, git…)
+  # outside the sandbox. Required because the sandbox glibc/ABI is
+  # incompatible with arbitrary host shared libraries — we route py_func and
+  # lua_func through the host instead.
+  - --talk-name=org.freedesktop.Flatpak
 
 build-options:
   build-args:

release_note.txt

@@ -16,6 +16,14 @@ version 0.1.3
   path, default report/log dirs, python/lua interpreter pickers) now
   bypasses the XDG document portal — the v0.1.2 fix was only on the
   "open test" dialog.
+- Flatpak: py_func / lua_func / run sub-instance now execute on the host
+  via flatpak-spawn, lifting the previous glibc/ABI incompatibility that
+  prevented user-configured host Python or Lua interpreters from being
+  reached from the sandbox.
+- Validation suite: single entry point with ``--mode source|wheel|
+  pyinstaller|flatpak|appimage`` to validate any packaging channel
+  against the same item set; reports are stamped per mode.
+- GUI: the "Run tum" test item now uses the testium logo.
 
 version 0.1.2
 ==============

src/testium/interpreter/test_items/test_item_run.py

@@ -24,9 +24,15 @@ def _testium_launch_cmd():
     appimage = os.environ.get("APPIMAGE")
     if appimage:
         return [appimage]
-    # Flatpak: re-launch via the Flatpak app id.
+    # Flatpak: re-launch via the Flatpak app id, but on the host side —
+    # the `flatpak` CLI cannot run inside our sandbox (no D-Bus access to the
+    # host Flatpak service, and the host binary would need host libs that are
+    # ABI-incompatible with the sandbox runtime). flatpak-spawn proxies the
+    # call to the host via org.freedesktop.Flatpak (allowed by --talk-name in
+    # the manifest).
     if os.path.isfile("/.flatpak-info"):
-        return ["flatpak", "run", "org.testium.Testium"]
+        return ["flatpak-spawn", "--host",
+                "flatpak", "run", "org.testium.Testium"]
     # PyInstaller frozen exe: sys.executable is the binary itself.
     if getattr(sys, "frozen", False):
         return [sys.executable]

src/testium/interpreter/utils/bins.py

@@ -17,8 +17,11 @@ Public API
 ``reset()``             : clear the cache (mostly useful for tests)
 """
 
+import atexit
 import os
+import shutil
 import subprocess
+import tempfile
 
 import api.testium as tm
 from interpreter.utils.paths import sys_app_path_lin, sys_app_path_win
@@ -30,20 +33,6 @@ from runtime.tum_except import ETUMRuntimeError
 _PYTHON_CANDIDATES = ["python3", "python"]
 _LUA_CANDIDATES = ["lua", "lua5.5", "lua5.4", "lua5.3", "lua5.2", "lua5.1"]
 
-# When running inside a Flatpak, --filesystem=host-os mounts the host at
-# /run/host (read-only). Binaries and libraries from the host are not on the
-# sandbox PATH/LD_LIBRARY_PATH, so we probe and inject them explicitly.
-_FLATPAK_HOST_DIRS = [
-    "/run/host/usr/local/bin",
-    "/run/host/usr/bin",
-    "/run/host/bin",
-]
-_FLATPAK_HOST_LIB_DIRS = [
-    "/run/host/usr/lib",
-    "/run/host/usr/lib64",
-    "/run/host/usr/local/lib",
-]
-
 # Inside an AppImage, AppRun prepends $APPDIR/usr/bin to PATH and exports a
 # bundle-local PYTHONHOME / PYTHONPATH / LD_LIBRARY_PATH. We want py_func and
 # lua_func to run under the *host* interpreter (not the bundled one), so we
@@ -64,50 +53,17 @@ def _in_appimage():
     return "APPIMAGE" in os.environ
 
 
-def apply_host_lua_paths(env):
-    """Prepend host Lua module dirs to LUA_PATH / LUA_CPATH (Flatpak only).
-
-    Must be called after user-defined lua_env overrides are applied, so host
-    paths are always first regardless of user config. User-defined paths remain
-    in the variable but after the host ones.
-    """
-    if not _in_flatpak():
-        return
-    _LUA_VERSIONS = ["5.5", "5.4", "5.3", "5.2", "5.1"]
-    _HOST = "/run/host/usr"
-    cpath_dirs, lpath_dirs = [], []
-    for v in _LUA_VERSIONS:
-        for base in [f"{_HOST}/lib/lua/{v}",
-                     f"{_HOST}/lib64/lua/{v}",
-                     f"{_HOST}/lib/x86_64-linux-gnu/lua/{v}"]:
-            cpath_dirs.append(f"{base}/?.so")
-        lpath_dirs.append(f"{_HOST}/share/lua/{v}/?.lua")
-        lpath_dirs.append(f"{_HOST}/share/lua/{v}/?/init.lua")
-    sep = ";"
-    host_cpath = sep.join(cpath_dirs)
-    host_lpath = sep.join(lpath_dirs)
-    # ;; keeps Lua's compiled-in defaults at the end as last resort
-    env["LUA_CPATH"] = host_cpath + sep + env.get("LUA_CPATH", ";;")
-    env["LUA_PATH"]  = host_lpath + sep + env.get("LUA_PATH",  ";;")
-
-
 def apply_host_libs(env):
-    """Prepare *env* for launching a host binary from inside our bundle.
+    """Strip bundle-local entries from *env* so a host binary can run cleanly.
 
-    - Flatpak: prepend host library dirs to LD_LIBRARY_PATH so the dynamic
+    Only meaningful for AppImage: removes $APPDIR-prefixed entries from
-      linker can find host .so files mounted under /run/host.
+    LD_LIBRARY_PATH / PYTHONPATH / PATH and drops PYTHONHOME, so the host
-    - AppImage: strip $APPDIR-prefixed entries from LD_LIBRARY_PATH and
+    interpreter doesn't try to load the bundled (incompatible) Python
-      PYTHONPATH and drop PYTHONHOME, so the host interpreter doesn't try
+    lib/site-packages. Flatpak is handled via flatpak-spawn --host instead
-      to load the bundled (incompatible) Python lib/site-packages.
+    (see flatpak_host_spawn), so the sandbox env is irrelevant there.
-    - Otherwise: no-op.
     """
-    if _in_flatpak():
+    if not _in_appimage():
-        dirs = ":".join(d for d in _FLATPAK_HOST_LIB_DIRS if os.path.isdir(d))
-        if dirs:
-            existing = env.get("LD_LIBRARY_PATH", "")
-            env["LD_LIBRARY_PATH"] = dirs + (":" + existing if existing else "")
         return
-    if _in_appimage():
     appdir = os.environ.get("APPDIR", "")
     if appdir:
         for var, sep in (("LD_LIBRARY_PATH", ":"),
@@ -127,15 +83,132 @@ def apply_host_libs(env):
     env.pop("PYTHONHOME", None)
 
 
+# ---------- Flatpak: spawn on host outside the sandbox -----------------------
+#
+# Inside a Flatpak the sandbox glibc is incompatible with host shared libraries,
+# so we can't run host Python/Lua under the sandbox runtime — `LD_LIBRARY_PATH`
+# tricks hit a `_dl_call_libc_early_init` assertion. The supported way out is
+# `flatpak-spawn --host`, which talks to the session-bus Flatpak D-Bus service
+# (org.freedesktop.Flatpak.Development) and asks it to spawn a process in the
+# host execution environment instead of inside our sandbox. The manifest must
+# grant `--talk-name=org.freedesktop.Flatpak` for the D-Bus call to be allowed.
+#
+# The host process can't see our /app/ contents (sandbox-only), so when we
+# spawn host Python/Lua to run `py_func` / `lua_func`, the cwd must be a
+# directory both sides can reach. /tmp is shared (--filesystem=/tmp), so we
+# stage the testium package there once per process and reuse it for every
+# spawn. In source mode (testium under $HOME) the host already sees the
+# original path, so we skip the copy.
+
+_staged_testium_path = None
+
+
+def _get_host_testium_path():
+    """Return a path to the testium package that the host can read.
+
+    - Source / wheel / PyInstaller install under $HOME → return testium_path()
+      as-is (host sees the same path via --filesystem=home).
+    - Flatpak bundle (testium under /app/) → stage a copy under /tmp on first
+      call and reuse it for the rest of the process.
+    """
+    global _staged_testium_path
+    if _staged_testium_path is not None:
+        return _staged_testium_path
+
+    # Imported lazily to avoid a circular import (paths.py -> api.testium).
+    from interpreter.utils.paths import testium_path
+    tp = testium_path()
+
+    if not tp.startswith("/app/"):
+        _staged_testium_path = tp
+        return tp
+
+    staged = tempfile.mkdtemp(prefix="testium_host_", dir="/tmp")
+    # copytree refuses to write into an existing dir unless dirs_exist_ok=True.
+    # mkdtemp creates the dir, so we copy *into* it.
+    for entry in os.listdir(tp):
+        src = os.path.join(tp, entry)
+        dst = os.path.join(staged, entry)
+        if os.path.isdir(src):
+            shutil.copytree(src, dst, symlinks=True)
+        else:
+            shutil.copy2(src, dst, follow_symlinks=False)
+    _staged_testium_path = staged
+    atexit.register(shutil.rmtree, staged, ignore_errors=True)
+    return staged
+
+
+_FORWARDED_ENV_KEYS = (
+    "HOME", "USER", "LOGNAME", "TMPDIR",
+    "XDG_RUNTIME_DIR", "XDG_DATA_HOME", "XDG_CONFIG_HOME", "XDG_CACHE_HOME",
+    "DBUS_SESSION_BUS_ADDRESS", "DISPLAY", "WAYLAND_DISPLAY",
+    "LANG", "LC_ALL",
+)
+
+
+def flatpak_host_spawn(interp_bin, cmd_args, host_cwd, extra_env=None):
+    """Build a flatpak-spawn --host command vector.
+
+    Args:
+        interp_bin: absolute path to the host interpreter (e.g. /usr/bin/python3).
+        cmd_args: list of arguments passed to the interpreter.
+        host_cwd: working directory on the host (must be reachable from host).
+        extra_env: optional {name: value} of env vars to set on the host side
+                   in addition to the default forwarded set. Values of ""
+                   unset the variable on the host.
+
+    Returns a list suitable for subprocess.Popen.
+    """
+    spawn = ["flatpak-spawn", "--host", f"--directory={host_cwd}"]
+    forwarded = {}
+    for key in _FORWARDED_ENV_KEYS:
+        val = os.environ.get(key)
+        if val:
+            forwarded[key] = val
+    if extra_env:
+        forwarded.update(extra_env)
+    for k, v in forwarded.items():
+        if v == "":
+            spawn.append(f"--unset-env={k}")
+        else:
+            spawn.append(f"--env={k}={v}")
+    spawn.append(interp_bin)
+    spawn.extend(cmd_args)
+    return spawn
+
+
+def _which_host_flatpak(name):
+    """Resolve a binary name (or absolute path) on the host via flatpak-spawn.
+
+    We can't probe /run/host/... because (a) only host-os is mounted there,
+    not arbitrary paths like /scratch, and (b) returning a /run/host path
+    would be useless — the host-side spawn sees a different filesystem and
+    needs the host-native path anyway.
+    """
+    if os.path.isabs(name):
+        cmd = flatpak_host_spawn("/bin/sh", ["-c", f'test -x "{name}"'],
+                                 host_cwd="/tmp")
+        try:
+            r = subprocess.run(cmd, capture_output=True, timeout=10)
+        except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired):
+            return ""
+        return name if r.returncode == 0 else ""
+    cmd = flatpak_host_spawn("/bin/sh", ["-c", f'command -v "{name}"'],
+                             host_cwd="/tmp")
+    try:
+        r = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
+    except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired):
+        return ""
+    if r.returncode != 0:
+        return ""
+    return r.stdout.strip()
+
+
 def _which(name):
     if tm.OS() == "Windows":
         return sys_app_path_win(name)
     if _in_flatpak():
-        for d in _FLATPAK_HOST_DIRS:
+        return _which_host_flatpak(name)
-            p = os.path.join(d, name)
-            if os.path.isfile(p) and os.access(p, os.X_OK):
-                return p
-        return ""
     if _in_appimage():
         for d in _APPIMAGE_HOST_DIRS:
             p = os.path.join(d, name)
@@ -146,14 +219,33 @@ def _which(name):
 
 
 def _probe_env():
-    """Subprocess env for probing host binaries (adds host libs in Flatpak)."""
+    """Subprocess env for probing host binaries.
+
+    In AppImage we still need to scrub APPDIR-prefixed entries; in Flatpak we
+    delegate execution to the host via flatpak-spawn so the sandbox env doesn't
+    matter, but apply_host_libs is a no-op cost.
+    """
     env = os.environ.copy()
     apply_host_libs(env)
     return env
 
 
-def _python_version(path):
+def _run_probe(cmd):
-    cmd = [path, "-c", "import sys; print(sys.version_info[:3])"]
+    """Run a probe command, dispatching through flatpak-spawn --host in Flatpak.
+
+    Returns (stdout, stderr) as str, or None on failure.
+    """
+    if _in_flatpak():
+        spawn = flatpak_host_spawn(cmd[0], cmd[1:], host_cwd="/tmp")
+        try:
+            r = subprocess.run(
+                spawn, capture_output=True, text=True, timeout=10,
+            )
+        except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired):
+            return None
+        if r.returncode != 0:
+            return None
+        return r.stdout, r.stderr
     try:
         r = subprocess.run(
             cmd, capture_output=True, text=True,
@@ -161,8 +253,15 @@ def _python_version(path):
         )
     except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired):
         return None
+    return r.stdout, r.stderr
+
+
+def _python_version(path):
+    out = _run_probe([path, "-c", "import sys; print(sys.version_info[:3])"])
+    if out is None:
+        return None
     try:
-        return eval(r.stdout)
+        return eval(out[0])
     except Exception:
         return None
 
@@ -173,15 +272,11 @@ def _is_python3(path):
 
 
 def _lua_version(path):
-    try:
+    out = _run_probe([path, "-v"])
-        r = subprocess.run(
+    if out is None:
-            [path, "-v"], capture_output=True, text=True, timeout=10,
-            env=_probe_env(),
-        )
-    except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired):
         return None
     # On Windows the version banner goes to stderr.
-    line = r.stdout or r.stderr
+    line = out[0] or out[1]
     try:
         major, minor, _patch = line.split(" ")[1].split(".")
         return (int(major), int(minor))
@@ -225,7 +320,10 @@ def _resolve(name):
         # Absolute path: accept as-is (user knows exactly what they want).
         # Bare name: resolve via _which() so the override stays host-only in
         # Flatpak/AppImage instead of silently picking the bundled interpreter.
-        if os.path.isabs(override):
+        # In Flatpak we always defer to _which() so even absolute paths are
+        # checked from the host's perspective (the sandbox can't see e.g.
+        # /scratch/... paths that the user may have configured).
+        if os.path.isabs(override) and not _in_flatpak():
             resolved = override if (os.path.isfile(override)
                                     and os.access(override, os.X_OK)) else ""
         else:

src/testium/interpreter/utils/lua_process.py

@@ -47,6 +47,13 @@ class LuaProcessBase:
         if self._process is not None:
             raise ETUMRuntimeError("The function subprocess has already been started.")
 
+        # In Flatpak the host can't see /app/lib/testium/lua_func, so use a
+        # staged copy under /tmp (shared between sandbox and host).
+        if bins._in_flatpak():
+            func_proc_path = os.path.join(
+                bins._get_host_testium_path(), "lua_func"
+            )
+        else:
             func_proc_path = os.path.realpath(
                 os.path.join(subproc_path(), "lua_func")
             )
@@ -71,7 +78,6 @@ class LuaProcessBase:
                     env[k] = e
                 else:
                     env[k] = e + ";" + env.get(k, "")
-        bins.apply_host_lua_paths(env)
 
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         sock.bind(("localhost", 0))
@@ -79,8 +85,7 @@ class LuaProcessBase:
         sock.close()
 
         # POpen params
-        params = [
+        cmd_args = [
-            self._lbin,
             "main.lua",
             "--timeout",
             f"{self._timeout}",
@@ -91,14 +96,31 @@ class LuaProcessBase:
         ]
 
         if tm.debug_enabled() and tm.gd("debug_rpc", False):
-            params.append("--verbose")
+            cmd_args.append("--verbose")
+
+        if bins._in_flatpak():
+            # Run on the host outside the sandbox: avoids glibc ABI mismatches
+            # between the Flatpak runtime and host shared libraries.
+            host_env = {
+                k: env[k] for k in ("LUA_PATH", "LUA_CPATH", "PATH")
+                if k in env and env[k]
+            }
+            params = bins.flatpak_host_spawn(
+                self._lbin, cmd_args, host_cwd=func_proc_path,
+                extra_env=host_env,
+            )
+            popen_kwargs = {}
+        else:
+            params = [self._lbin, *cmd_args]
+            popen_kwargs = {"env": env, "cwd": func_proc_path}
 
         self._process = subprocess.Popen(
-            params, env=env, cwd=func_proc_path,
+            params,
             stdin=subprocess.DEVNULL,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             restore_signals=False,
+            **popen_kwargs,
         )
         # Route subprocess stdout/stderr (lua require failures, syntax
         # errors, anything written to fd 1/2 before the in-script

src/testium/interpreter/utils/py_process.py

@@ -61,14 +61,18 @@ class PyProcessBase:
         if sock is not None:
             sock.close()
 
-        # Add the path of the subprocess (root sources of testium)
+        # In Flatpak the host can't see /app/lib/testium, so use a staged copy
+        # under /tmp (shared between sandbox and host) for both cwd and as the
+        # root in PYTHONPATH. Outside Flatpak the original paths are used.
+        if bins._in_flatpak():
+            tstium_path = bins._get_host_testium_path()
+            func_proc_path = tstium_path
+        else:
             tstium_path = os.path.realpath(testium_path())
             func_proc_path = os.path.realpath(subproc_path())
         env["PYTHONPATH"] = tstium_path + os.pathsep + self._ppath + os.pathsep + env.get("PYTHONPATH", "")
 
-        params = [
+        cmd_args = [
-            self._pbin,
-            # "-m",
             "py_func",
             "-p",
             f"{self._port}",
@@ -77,14 +81,31 @@ class PyProcessBase:
         ]
 
         if tm.debug_enabled() and tm.gd("debug_rpc", False):
-            params.append("-v")
+            cmd_args.append("-v")
+
+        if bins._in_flatpak():
+            # Run on the host outside the sandbox: avoids glibc ABI mismatches
+            # between the Flatpak runtime and host shared libraries.
+            host_env = {
+                k: env[k] for k in ("PYTHONPATH", "PATH")
+                if k in env and env[k]
+            }
+            params = bins.flatpak_host_spawn(
+                self._pbin, cmd_args, host_cwd=func_proc_path,
+                extra_env=host_env,
+            )
+            popen_kwargs = {}
+        else:
+            params = [self._pbin, *cmd_args]
+            popen_kwargs = {"env": env, "cwd": func_proc_path}
 
         self._process = subprocess.Popen(
-            params, env=env, cwd=func_proc_path,
+            params,
             stdin=subprocess.DEVNULL,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             restore_signals=False,
+            **popen_kwargs,
         )
         # Route subprocess stdout/stderr (early-startup errors,
         # unhandled exceptions, anything written to fd 1/2 before the

test/validation/README.md

@@ -1,34 +1,67 @@
 # Validation
 
-This directory contains the testium validation suite.
+This directory contains the testium validation suite. A single set of
+items (`items/`), fixtures and post-processing (`post_execution.py`) is
+re-used across every packaging channel.
 
 ## Running the suite
 
 ```sh
-./test/validation/run.sh           # Linux
+./test/validation/run.sh                       # default mode = source
-test\validation\run.bat            # Windows
+./test/validation/run.sh --mode wheel
+./test/validation/run.sh --mode pyinstaller
+./test/validation/run.sh --mode flatpak
+./test/validation/run.sh --mode appimage
 ```
 
-The wrapper creates a dedicated Python venv in the system temp dir
+On Windows (only `source`, `wheel`, `pyinstaller` are supported):
-(`${TMPDIR:-/tmp}/testium-validation-venv` on Linux, `%TEMP%\testium-validation-venv`
-on Windows), using `--system-site-packages` so existing system packages
-stay visible. The validation suite is then run with that venv pinned as
-`python_bin`. Every test-execution subprocess (inline `<| ... |>`
-evaluation, `py_func`, `cycle`, `post_execution`, ...) runs inside the
-venv, while testium itself keeps running in the project's own
-environment.
 
-Pass `clean` as the first argument to recreate the venv from scratch
+```bat
-(useful after a system Python upgrade):
+test\validation\run.bat --mode pyinstaller
+```
+
+Pass `clean` as the **first** argument to recreate the validation venv
+from scratch (useful after a system Python upgrade):
 
 ```sh
-./test/validation/run.sh clean
+./test/validation/run.sh clean --mode flatpak
 ```
 
+Any extra arguments after the mode flag are forwarded to testium.
+
+## Modes
+
+| Mode          | What it launches                                            | Prerequisite                                                     |
+|---------------|-------------------------------------------------------------|------------------------------------------------------------------|
+| `source`      | `python3 src/testium` via the project's `run.sh`            | none — works straight out of the repo                            |
+| `wheel`       | `python -m testium` inside a dedicated wheel venv           | `./build_all.sh` produced `dist/testium-<v>-py3-none-any.whl`    |
+| `pyinstaller` | `dist/testium-<v>` (frozen binary)                          | `./build_all.sh` produced the PyInstaller binary                 |
+| `flatpak`     | `flatpak run --command=testium org.testium.Testium`         | the Flatpak bundle is installed (`flatpak install --user dist/testium-<v>.flatpak`) |
+| `appimage`    | `dist/Testium-<v>-x86_64.AppImage`                          | `./build_all.sh` produced the AppImage                           |
+
+Each mode writes its results to a distinct report file
+(`validation-<mode>.sqlite` / `validation-<mode>-<item>.xml`), so you
+can run several modes in a row without clobbering previous reports.
+
+## How `python_bin` is pinned
+
+Every test-execution subprocess (inline `<| ... |>` evaluation,
+`py_func`, `cycle`, `post_execution`, …) is routed through a dedicated
+venv at `${TMPDIR:-/tmp}/testium-validation-venv`. The venv is created
+with `--system-site-packages` so existing system packages stay visible,
+then `junit-xml` is pip-installed for `post_execution.py`.
+
+This is a **host** venv. In every mode (including Flatpak) the
+test-execution subprocesses end up running on the host — directly for
+source/wheel/pyinstaller/appimage, and via `flatpak-spawn --host` for
+Flatpak — so the same venv works across modes. The wheel mode
+additionally creates a separate `testium-wheel-venv-<v>` to hold the
+installed wheel; that one is only used to launch testium itself.
+
 ## What is checked
 
-The `venv` item under `items/venv/` asserts that the venv is actually
+The `venv` item under `items/venv/` asserts that the validation venv is
-being used:
+actually being used:
 
 * `python_bin` is set in the global dict.
 * The eval subprocess (used for `<| ... |>` expressions) has

test/validation/run.bat

@@ -1,61 +1,131 @@
 @echo off
-SETLOCAL EnableExtensions
+SETLOCAL EnableExtensions EnableDelayedExpansion
 
-REM Runs the testium validation suite with a dedicated Python venv used
+REM Runs the testium validation suite against any installable channel of
-REM by every py_func / cycle / inline-eval subprocess. testium itself
+REM testium on Windows (source, wheel, pyinstaller).
-REM keeps running in the project's own environment; the validation venv
-REM only isolates *test execution*.
 REM
-REM   test\validation\run.bat [clean] [extra testium args]
+REM Usage:
+REM   test\validation\run.bat [clean] [--mode MODE] [extra testium args]
 REM
-REM Requires the project venv to already exist (run the project's
+REM   clean       remove the validation venv before recreating it
-REM run.bat once first, or any other testium install method).
+REM               (must be the first argument; useful after a Python upgrade)
+REM
+REM   --mode MODE which testium build to validate. One of:
+REM                   source       (default) project's run.bat (src\testium)
+REM                   wheel        dist\testium-<v>-py3-none-any.whl
+REM                   pyinstaller  dist\testium-<v>.exe (or dist\testium-<v>)
+REM
+REM Every test-execution subprocess runs in a dedicated host venv under
+REM %TEMP%\testium-validation-venv (created with --system-site-packages,
+REM then junit-xml is pip-installed for post_execution.py).
+REM
+REM The report file is suffixed with the mode so consecutive runs in
+REM different modes don't overwrite each other.
 
 SET "SCRIPT_DIR=%~dp0"
-SET "PROJECT_DIR=%SCRIPT_DIR%..\.."
+IF "%SCRIPT_DIR:~-1%"=="\" SET "SCRIPT_DIR=%SCRIPT_DIR:~0,-1%"
-REM Venv in the user temp dir (Windows equivalent of /tmp).
+SET "PROJECT_DIR=%SCRIPT_DIR%\..\.."
-SET "VENV_DIR=%TEMP%\testium-validation-venv"
+SET /P VERSION=<"%PROJECT_DIR%\src\VERSION"
-SET "PROJECT_VENV=%PROJECT_DIR%\test\tmp\testium_venv"
 
+REM ---------- arg parsing ----------------------------------------------------
+
+SET "MODE=source"
+SET "CLEAN=0"
 IF /I "%~1"=="clean" (
-    rmdir /s /q "%VENV_DIR%"
+    SET "CLEAN=1"
     SHIFT
 )
 
-REM Locate a host Python.
+SET "EXTRA="
-SET "PYTHON_EXE=python"
+:PARSE_ARGS
+IF "%~1"=="" GOTO ARGS_DONE
+IF /I "%~1"=="--mode" (
+    SET "MODE=%~2"
+    SHIFT
+    SHIFT
+    GOTO PARSE_ARGS
+)
+SET "EXTRA=!EXTRA! "%~1""
+SHIFT
+GOTO PARSE_ARGS
+:ARGS_DONE
+
+REM ---------- locate host python ---------------------------------------------
+
+SET "PYTHON_EXE="
 py --version >nul 2>&1
 IF %ERRORLEVEL% EQU 0 (
     SET "PYTHON_EXE=py"
-    goto :PYTHON_FOUND
+    GOTO PYTHON_FOUND
 )
 python --version >nul 2>&1
 IF %ERRORLEVEL% EQU 0 (
     SET "PYTHON_EXE=python"
-    goto :PYTHON_FOUND
+    GOTO PYTHON_FOUND
 )
-echo ERROR : Python could not be found on this system.
+echo ERROR: Python could not be found on this system.
 exit /b 1
-
 :PYTHON_FOUND
 
+REM ---------- validation venv -------------------------------------------------
+
+SET "VENV_DIR=%TEMP%\testium-validation-venv"
+IF "%CLEAN%"=="1" IF EXIST "%VENV_DIR%" rmdir /s /q "%VENV_DIR%"
+
 IF NOT EXIST "%VENV_DIR%" (
     echo Creating validation venv at %VENV_DIR%
     %PYTHON_EXE% -m venv --system-site-packages "%VENV_DIR%"
-    IF %ERRORLEVEL% NEQ 0 (
+    IF !ERRORLEVEL! NEQ 0 (
         echo ERROR while creating the validation venv.
         exit /b 1
     )
     call "%VENV_DIR%\Scripts\pip" install --quiet --upgrade pip
     call "%VENV_DIR%\Scripts\pip" install --quiet junit-xml
 )
-
 SET "VENV_PYTHON=%VENV_DIR%\Scripts\python.exe"
 
-IF NOT EXIST "%PROJECT_VENV%" (
+REM ---------- shared "tail" forwarded to every launcher -----------------------
-    echo ERROR : project venv not found at %PROJECT_VENV%. Run the project run.bat once first.
+REM Reports are stamped with the mode so successive runs don't clobber each other.
+
+SET "TAIL=-b -d "python_bin=%VENV_PYTHON%" -d "validation_report_file=validation-%MODE%" -- "%SCRIPT_DIR%\main.tum"%EXTRA%"
+
+REM ---------- per-mode launcher ----------------------------------------------
+
+echo -- validation mode: %MODE%
+
+IF /I "%MODE%"=="source"      GOTO MODE_SOURCE
+IF /I "%MODE%"=="wheel"       GOTO MODE_WHEEL
+IF /I "%MODE%"=="pyinstaller" GOTO MODE_PYI
+echo ERROR: unknown --mode '%MODE%'. Expected: source ^| wheel ^| pyinstaller.
+exit /b 1
+
+:MODE_SOURCE
+call "%PROJECT_DIR%\run.bat" %TAIL%
+exit /b %ERRORLEVEL%
+
+:MODE_WHEEL
+SET "WHEEL=%PROJECT_DIR%\dist\testium-%VERSION%-py3-none-any.whl"
+IF NOT EXIST "%WHEEL%" (
+    echo ERROR: wheel not found at %WHEEL% -- run build_all.sh first.
     exit /b 1
 )
+SET "WHEEL_VENV=%TEMP%\testium-wheel-venv-%VERSION%"
+IF "%CLEAN%"=="1" IF EXIST "%WHEEL_VENV%" rmdir /s /q "%WHEEL_VENV%"
+IF NOT EXIST "%WHEEL_VENV%" (
+    echo Creating wheel venv at %WHEEL_VENV%
+    %PYTHON_EXE% -m venv --system-site-packages "%WHEEL_VENV%"
+    call "%WHEEL_VENV%\Scripts\pip" install --quiet --upgrade pip
+    call "%WHEEL_VENV%\Scripts\pip" install --quiet "%WHEEL%"
+)
+"%WHEEL_VENV%\Scripts\python.exe" -m testium %TAIL%
+exit /b %ERRORLEVEL%
 
-call "%PROJECT_VENV%\Scripts\activate"
+:MODE_PYI
-python "%PROJECT_DIR%\src\testium" -b -d "python_bin=%VENV_PYTHON%" -- "%SCRIPT_DIR%main.tum" %*
+SET "PYI_BIN=%PROJECT_DIR%\dist\testium-%VERSION%.exe"
+IF NOT EXIST "%PYI_BIN%" SET "PYI_BIN=%PROJECT_DIR%\dist\testium-%VERSION%"
+IF NOT EXIST "%PYI_BIN%" (
+    echo ERROR: PyInstaller binary not found in %PROJECT_DIR%\dist -- run build_all.sh first.
+    exit /b 1
+)
+"%PYI_BIN%" %TAIL%
+exit /b %ERRORLEVEL%

test/validation/run.sh

@@ -1,47 +1,143 @@
 #!/bin/bash
-# Runs the testium validation suite with a dedicated Python venv used by
+# Runs the testium validation suite against any installable channel of
-# every py_func / cycle / inline-eval subprocess (i.e. everything that
+# testium (source, wheel, pyinstaller, flatpak, appimage).
-# goes through ``bins.python_bin()``). testium itself keeps running in
-# the project's own environment — the validation venv only isolates
-# *test execution*.
 #
-#   ./test/validation/run.sh [clean] [extra testium args]
+# Usage:
+#   ./test/validation/run.sh [clean] [--mode MODE] [extra testium args]
 #
-# ``clean`` (optional, must be the first arg) removes the venv before
+#   clean           remove the validation venv before recreating it
-# recreating it; this is the way to refresh the venv after a system
+#                   (must be the first argument; useful after a Python upgrade)
-# Python upgrade.
+#
+#   --mode MODE     which testium build to validate. One of:
+#                       source       (default) src/testium via project run.sh
+#                       wheel        dist/testium-<v>-py3-none-any.whl
+#                       pyinstaller  dist/testium-<v>
+#                       flatpak      installed org.testium.Testium
+#                       appimage     dist/Testium-<v>-*.AppImage
+#
+# Every test-execution subprocess (inline <| ... |>, py_func, cycle,
+# post_execution, ...) runs in a dedicated host venv under
+# /tmp/testium-validation-venv. That venv is shared across modes —
+# even Flatpak reaches it via flatpak-spawn --host. The validation venv
+# is created with --system-site-packages so existing system packages
+# (PySide6, lxml, ...) stay visible, then junit-xml is pip-installed
+# for post_execution.py.
+#
+# The report file is suffixed with the mode (e.g. validation-flatpak.sqlite)
+# so consecutive runs in different modes don't overwrite each other.
 
 set -e
 
 SCRIPT_PATH="$(readlink -f "$0")"
 SCRIPT_DIR="$(realpath "$(dirname "$SCRIPT_PATH")")"
 PROJECT_DIR="$(realpath "$SCRIPT_DIR/../..")"
-# Venv lives in the system temp dir so it stays out of the project tree
+VERSION="$(cat "$PROJECT_DIR/src/VERSION")"
-# (and is naturally cleaned up by tmpfiles/reboot on most distros).
+
-VENV_DIR="${TMPDIR:-/tmp}/testium-validation-venv"
+# ---------- arg parsing -------------------------------------------------------
+
+MODE=source
 
 if [ "${1:-}" = "clean" ]; then
-    rm -rf "$VENV_DIR"
+    CLEAN=1
     shift
+else
+    CLEAN=0
+fi
+
+EXTRA=()
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --mode)
+            MODE="$2"
+            shift 2
+            ;;
+        --mode=*)
+            MODE="${1#--mode=}"
+            shift
+            ;;
+        *)
+            EXTRA+=("$1")
+            shift
+            ;;
+    esac
+done
+
+# ---------- validation venv ---------------------------------------------------
+
+VENV_DIR="${TMPDIR:-/tmp}/testium-validation-venv"
+if [ "$CLEAN" -eq 1 ]; then
+    rm -rf "$VENV_DIR"
 fi
 
 if [ ! -d "$VENV_DIR" ]; then
     echo "Creating validation venv at $VENV_DIR"
-    # --system-site-packages so we don't have to reinstall pyside6, lxml
-    # & friends just to support the validation helpers. We still pip
-    # install junit-xml below because it is the one dep that does *not*
-    # ship as a system package on most distros and is required by
-    # post_execution.py.
     python3 -m venv --system-site-packages "$VENV_DIR"
     "$VENV_DIR/bin/pip" install --quiet --upgrade pip
     "$VENV_DIR/bin/pip" install --quiet junit-xml
 fi
-
 VENV_PYTHON="$VENV_DIR/bin/python3"
 
-# Delegate to the project's run.sh so testium itself still runs in the
+# ---------- per-mode launcher -------------------------------------------------
-# project venv (with pyside6, gitpython, ...). ``-d python_bin=...``
+
-# pins every test-execution subprocess to the validation venv.
+case "$MODE" in
-exec "$PROJECT_DIR/run.sh" -b \
+    source)
+        CMD=("$PROJECT_DIR/run.sh")
+        ;;
+    wheel)
+        WHEEL="$PROJECT_DIR/dist/testium-${VERSION}-py3-none-any.whl"
+        if [ ! -f "$WHEEL" ]; then
+            echo "ERROR: wheel not found at $WHEEL — run ./build_all.sh first." >&2
+            exit 1
+        fi
+        WHEEL_VENV="${TMPDIR:-/tmp}/testium-wheel-venv-${VERSION}"
+        if [ "$CLEAN" -eq 1 ]; then
+            rm -rf "$WHEEL_VENV"
+        fi
+        if [ ! -d "$WHEEL_VENV" ]; then
+            echo "Creating wheel venv at $WHEEL_VENV"
+            python3 -m venv --system-site-packages "$WHEEL_VENV"
+            "$WHEEL_VENV/bin/pip" install --quiet --upgrade pip
+            "$WHEEL_VENV/bin/pip" install --quiet "$WHEEL"
+        fi
+        CMD=("$WHEEL_VENV/bin/python" -m testium)
+        ;;
+    pyinstaller)
+        PYI_BIN="$PROJECT_DIR/dist/testium-${VERSION}"
+        if [ ! -x "$PYI_BIN" ]; then
+            echo "ERROR: PyInstaller binary not found at $PYI_BIN — run ./build_all.sh first." >&2
+            exit 1
+        fi
+        CMD=("$PYI_BIN")
+        ;;
+    flatpak)
+        if ! flatpak info --user org.testium.Testium &>/dev/null \
+           && ! flatpak info --system org.testium.Testium &>/dev/null; then
+            echo "ERROR: org.testium.Testium is not installed." >&2
+            echo "       flatpak install --user $PROJECT_DIR/dist/testium-${VERSION}.flatpak" >&2
+            exit 1
+        fi
+        CMD=(flatpak run --command=testium org.testium.Testium)
+        ;;
+    appimage)
+        APPIMAGE=$(ls -1t "$PROJECT_DIR/dist"/Testium-"${VERSION}"-*.AppImage 2>/dev/null | head -1)
+        if [ -z "$APPIMAGE" ] || [ ! -x "$APPIMAGE" ]; then
+            echo "ERROR: no AppImage for version $VERSION under $PROJECT_DIR/dist — run ./build_all.sh first." >&2
+            exit 1
+        fi
+        CMD=("$APPIMAGE")
+        ;;
+    *)
+        echo "ERROR: unknown --mode '$MODE'. Expected: source|wheel|pyinstaller|flatpak|appimage." >&2
+        exit 1
+        ;;
+esac
+
+# ---------- launch ------------------------------------------------------------
+
+echo "-- validation mode: $MODE"
+echo "-- launch: ${CMD[*]}"
+
+exec "${CMD[@]}" -b \
     -d "python_bin=$VENV_PYTHON" \
-    -- "$SCRIPT_DIR/main.tum" "$@"
+    -d "validation_report_file=validation-$MODE" \
+    -- "$SCRIPT_DIR/main.tum" "${EXTRA[@]}"