diff --git a/doc/manual/testium_manual.pdf b/doc/manual/testium_manual.pdf index c03d73c..dd483b4 100644 Binary files a/doc/manual/testium_manual.pdf and b/doc/manual/testium_manual.pdf differ diff --git a/package/flatpak/org.testium.Testium.yaml b/package/flatpak/org.testium.Testium.yaml index 68d2c04..f4e5cff 100644 --- a/package/flatpak/org.testium.Testium.yaml +++ b/package/flatpak/org.testium.Testium.yaml @@ -16,6 +16,11 @@ finish-args: - --filesystem=home - --filesystem=/tmp - --filesystem=host-os + # Allow flatpak-spawn --host to launch host binaries (Python, Lua, git…) + # outside the sandbox. Required because the sandbox glibc/ABI is + # incompatible with arbitrary host shared libraries — we route py_func and + # lua_func through the host instead. + - --talk-name=org.freedesktop.Flatpak build-options: build-args: diff --git a/release_note.txt b/release_note.txt index d15f3a6..08522a8 100644 --- a/release_note.txt +++ b/release_note.txt @@ -16,6 +16,14 @@ version 0.1.3 path, default report/log dirs, python/lua interpreter pickers) now bypasses the XDG document portal — the v0.1.2 fix was only on the "open test" dialog. +- Flatpak: py_func / lua_func / run sub-instance now execute on the host + via flatpak-spawn, lifting the previous glibc/ABI incompatibility that + prevented user-configured host Python or Lua interpreters from being + reached from the sandbox. +- Validation suite: single entry point with ``--mode source|wheel| + pyinstaller|flatpak|appimage`` to validate any packaging channel + against the same item set; reports are stamped per mode. +- GUI: the "Run tum" test item now uses the testium logo. version 0.1.2 ============== diff --git a/src/testium/interpreter/test_items/test_item_run.py b/src/testium/interpreter/test_items/test_item_run.py index a563f4c..b53b37c 100644 --- a/src/testium/interpreter/test_items/test_item_run.py +++ b/src/testium/interpreter/test_items/test_item_run.py @@ -24,9 +24,15 @@ def _testium_launch_cmd(): appimage = os.environ.get("APPIMAGE") if appimage: return [appimage] - # Flatpak: re-launch via the Flatpak app id. + # Flatpak: re-launch via the Flatpak app id, but on the host side — + # the `flatpak` CLI cannot run inside our sandbox (no D-Bus access to the + # host Flatpak service, and the host binary would need host libs that are + # ABI-incompatible with the sandbox runtime). flatpak-spawn proxies the + # call to the host via org.freedesktop.Flatpak (allowed by --talk-name in + # the manifest). if os.path.isfile("/.flatpak-info"): - return ["flatpak", "run", "org.testium.Testium"] + return ["flatpak-spawn", "--host", + "flatpak", "run", "org.testium.Testium"] # PyInstaller frozen exe: sys.executable is the binary itself. if getattr(sys, "frozen", False): return [sys.executable] diff --git a/src/testium/interpreter/utils/bins.py b/src/testium/interpreter/utils/bins.py index e553252..cdf4c6e 100644 --- a/src/testium/interpreter/utils/bins.py +++ b/src/testium/interpreter/utils/bins.py @@ -17,8 +17,11 @@ Public API ``reset()`` : clear the cache (mostly useful for tests) """ +import atexit import os +import shutil import subprocess +import tempfile import api.testium as tm from interpreter.utils.paths import sys_app_path_lin, sys_app_path_win @@ -30,20 +33,6 @@ from runtime.tum_except import ETUMRuntimeError _PYTHON_CANDIDATES = ["python3", "python"] _LUA_CANDIDATES = ["lua", "lua5.5", "lua5.4", "lua5.3", "lua5.2", "lua5.1"] -# When running inside a Flatpak, --filesystem=host-os mounts the host at -# /run/host (read-only). Binaries and libraries from the host are not on the -# sandbox PATH/LD_LIBRARY_PATH, so we probe and inject them explicitly. -_FLATPAK_HOST_DIRS = [ - "/run/host/usr/local/bin", - "/run/host/usr/bin", - "/run/host/bin", -] -_FLATPAK_HOST_LIB_DIRS = [ - "/run/host/usr/lib", - "/run/host/usr/lib64", - "/run/host/usr/local/lib", -] - # Inside an AppImage, AppRun prepends $APPDIR/usr/bin to PATH and exports a # bundle-local PYTHONHOME / PYTHONPATH / LD_LIBRARY_PATH. We want py_func and # lua_func to run under the *host* interpreter (not the bundled one), so we @@ -64,78 +53,162 @@ def _in_appimage(): return "APPIMAGE" in os.environ -def apply_host_lua_paths(env): - """Prepend host Lua module dirs to LUA_PATH / LUA_CPATH (Flatpak only). - - Must be called after user-defined lua_env overrides are applied, so host - paths are always first regardless of user config. User-defined paths remain - in the variable but after the host ones. - """ - if not _in_flatpak(): - return - _LUA_VERSIONS = ["5.5", "5.4", "5.3", "5.2", "5.1"] - _HOST = "/run/host/usr" - cpath_dirs, lpath_dirs = [], [] - for v in _LUA_VERSIONS: - for base in [f"{_HOST}/lib/lua/{v}", - f"{_HOST}/lib64/lua/{v}", - f"{_HOST}/lib/x86_64-linux-gnu/lua/{v}"]: - cpath_dirs.append(f"{base}/?.so") - lpath_dirs.append(f"{_HOST}/share/lua/{v}/?.lua") - lpath_dirs.append(f"{_HOST}/share/lua/{v}/?/init.lua") - sep = ";" - host_cpath = sep.join(cpath_dirs) - host_lpath = sep.join(lpath_dirs) - # ;; keeps Lua's compiled-in defaults at the end as last resort - env["LUA_CPATH"] = host_cpath + sep + env.get("LUA_CPATH", ";;") - env["LUA_PATH"] = host_lpath + sep + env.get("LUA_PATH", ";;") - - def apply_host_libs(env): - """Prepare *env* for launching a host binary from inside our bundle. + """Strip bundle-local entries from *env* so a host binary can run cleanly. - - Flatpak: prepend host library dirs to LD_LIBRARY_PATH so the dynamic - linker can find host .so files mounted under /run/host. - - AppImage: strip $APPDIR-prefixed entries from LD_LIBRARY_PATH and - PYTHONPATH and drop PYTHONHOME, so the host interpreter doesn't try - to load the bundled (incompatible) Python lib/site-packages. - - Otherwise: no-op. + Only meaningful for AppImage: removes $APPDIR-prefixed entries from + LD_LIBRARY_PATH / PYTHONPATH / PATH and drops PYTHONHOME, so the host + interpreter doesn't try to load the bundled (incompatible) Python + lib/site-packages. Flatpak is handled via flatpak-spawn --host instead + (see flatpak_host_spawn), so the sandbox env is irrelevant there. """ - if _in_flatpak(): - dirs = ":".join(d for d in _FLATPAK_HOST_LIB_DIRS if os.path.isdir(d)) - if dirs: - existing = env.get("LD_LIBRARY_PATH", "") - env["LD_LIBRARY_PATH"] = dirs + (":" + existing if existing else "") + if not _in_appimage(): return - if _in_appimage(): - appdir = os.environ.get("APPDIR", "") - if appdir: - for var, sep in (("LD_LIBRARY_PATH", ":"), - ("PYTHONPATH", os.pathsep), - ("PATH", os.pathsep)): - cur = env.get(var, "") - if not cur: - continue - cleaned = sep.join( - p for p in cur.split(sep) - if p and not p.startswith(appdir) - ) - if cleaned: - env[var] = cleaned - else: - env.pop(var, None) - env.pop("PYTHONHOME", None) + appdir = os.environ.get("APPDIR", "") + if appdir: + for var, sep in (("LD_LIBRARY_PATH", ":"), + ("PYTHONPATH", os.pathsep), + ("PATH", os.pathsep)): + cur = env.get(var, "") + if not cur: + continue + cleaned = sep.join( + p for p in cur.split(sep) + if p and not p.startswith(appdir) + ) + if cleaned: + env[var] = cleaned + else: + env.pop(var, None) + env.pop("PYTHONHOME", None) + + +# ---------- Flatpak: spawn on host outside the sandbox ----------------------- +# +# Inside a Flatpak the sandbox glibc is incompatible with host shared libraries, +# so we can't run host Python/Lua under the sandbox runtime — `LD_LIBRARY_PATH` +# tricks hit a `_dl_call_libc_early_init` assertion. The supported way out is +# `flatpak-spawn --host`, which talks to the session-bus Flatpak D-Bus service +# (org.freedesktop.Flatpak.Development) and asks it to spawn a process in the +# host execution environment instead of inside our sandbox. The manifest must +# grant `--talk-name=org.freedesktop.Flatpak` for the D-Bus call to be allowed. +# +# The host process can't see our /app/ contents (sandbox-only), so when we +# spawn host Python/Lua to run `py_func` / `lua_func`, the cwd must be a +# directory both sides can reach. /tmp is shared (--filesystem=/tmp), so we +# stage the testium package there once per process and reuse it for every +# spawn. In source mode (testium under $HOME) the host already sees the +# original path, so we skip the copy. + +_staged_testium_path = None + + +def _get_host_testium_path(): + """Return a path to the testium package that the host can read. + + - Source / wheel / PyInstaller install under $HOME → return testium_path() + as-is (host sees the same path via --filesystem=home). + - Flatpak bundle (testium under /app/) → stage a copy under /tmp on first + call and reuse it for the rest of the process. + """ + global _staged_testium_path + if _staged_testium_path is not None: + return _staged_testium_path + + # Imported lazily to avoid a circular import (paths.py -> api.testium). + from interpreter.utils.paths import testium_path + tp = testium_path() + + if not tp.startswith("/app/"): + _staged_testium_path = tp + return tp + + staged = tempfile.mkdtemp(prefix="testium_host_", dir="/tmp") + # copytree refuses to write into an existing dir unless dirs_exist_ok=True. + # mkdtemp creates the dir, so we copy *into* it. + for entry in os.listdir(tp): + src = os.path.join(tp, entry) + dst = os.path.join(staged, entry) + if os.path.isdir(src): + shutil.copytree(src, dst, symlinks=True) + else: + shutil.copy2(src, dst, follow_symlinks=False) + _staged_testium_path = staged + atexit.register(shutil.rmtree, staged, ignore_errors=True) + return staged + + +_FORWARDED_ENV_KEYS = ( + "HOME", "USER", "LOGNAME", "TMPDIR", + "XDG_RUNTIME_DIR", "XDG_DATA_HOME", "XDG_CONFIG_HOME", "XDG_CACHE_HOME", + "DBUS_SESSION_BUS_ADDRESS", "DISPLAY", "WAYLAND_DISPLAY", + "LANG", "LC_ALL", +) + + +def flatpak_host_spawn(interp_bin, cmd_args, host_cwd, extra_env=None): + """Build a flatpak-spawn --host command vector. + + Args: + interp_bin: absolute path to the host interpreter (e.g. /usr/bin/python3). + cmd_args: list of arguments passed to the interpreter. + host_cwd: working directory on the host (must be reachable from host). + extra_env: optional {name: value} of env vars to set on the host side + in addition to the default forwarded set. Values of "" + unset the variable on the host. + + Returns a list suitable for subprocess.Popen. + """ + spawn = ["flatpak-spawn", "--host", f"--directory={host_cwd}"] + forwarded = {} + for key in _FORWARDED_ENV_KEYS: + val = os.environ.get(key) + if val: + forwarded[key] = val + if extra_env: + forwarded.update(extra_env) + for k, v in forwarded.items(): + if v == "": + spawn.append(f"--unset-env={k}") + else: + spawn.append(f"--env={k}={v}") + spawn.append(interp_bin) + spawn.extend(cmd_args) + return spawn + + +def _which_host_flatpak(name): + """Resolve a binary name (or absolute path) on the host via flatpak-spawn. + + We can't probe /run/host/... because (a) only host-os is mounted there, + not arbitrary paths like /scratch, and (b) returning a /run/host path + would be useless — the host-side spawn sees a different filesystem and + needs the host-native path anyway. + """ + if os.path.isabs(name): + cmd = flatpak_host_spawn("/bin/sh", ["-c", f'test -x "{name}"'], + host_cwd="/tmp") + try: + r = subprocess.run(cmd, capture_output=True, timeout=10) + except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired): + return "" + return name if r.returncode == 0 else "" + cmd = flatpak_host_spawn("/bin/sh", ["-c", f'command -v "{name}"'], + host_cwd="/tmp") + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=10) + except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired): + return "" + if r.returncode != 0: + return "" + return r.stdout.strip() def _which(name): if tm.OS() == "Windows": return sys_app_path_win(name) if _in_flatpak(): - for d in _FLATPAK_HOST_DIRS: - p = os.path.join(d, name) - if os.path.isfile(p) and os.access(p, os.X_OK): - return p - return "" + return _which_host_flatpak(name) if _in_appimage(): for d in _APPIMAGE_HOST_DIRS: p = os.path.join(d, name) @@ -146,14 +219,33 @@ def _which(name): def _probe_env(): - """Subprocess env for probing host binaries (adds host libs in Flatpak).""" + """Subprocess env for probing host binaries. + + In AppImage we still need to scrub APPDIR-prefixed entries; in Flatpak we + delegate execution to the host via flatpak-spawn so the sandbox env doesn't + matter, but apply_host_libs is a no-op cost. + """ env = os.environ.copy() apply_host_libs(env) return env -def _python_version(path): - cmd = [path, "-c", "import sys; print(sys.version_info[:3])"] +def _run_probe(cmd): + """Run a probe command, dispatching through flatpak-spawn --host in Flatpak. + + Returns (stdout, stderr) as str, or None on failure. + """ + if _in_flatpak(): + spawn = flatpak_host_spawn(cmd[0], cmd[1:], host_cwd="/tmp") + try: + r = subprocess.run( + spawn, capture_output=True, text=True, timeout=10, + ) + except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired): + return None + if r.returncode != 0: + return None + return r.stdout, r.stderr try: r = subprocess.run( cmd, capture_output=True, text=True, @@ -161,8 +253,15 @@ def _python_version(path): ) except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired): return None + return r.stdout, r.stderr + + +def _python_version(path): + out = _run_probe([path, "-c", "import sys; print(sys.version_info[:3])"]) + if out is None: + return None try: - return eval(r.stdout) + return eval(out[0]) except Exception: return None @@ -173,15 +272,11 @@ def _is_python3(path): def _lua_version(path): - try: - r = subprocess.run( - [path, "-v"], capture_output=True, text=True, timeout=10, - env=_probe_env(), - ) - except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired): + out = _run_probe([path, "-v"]) + if out is None: return None # On Windows the version banner goes to stderr. - line = r.stdout or r.stderr + line = out[0] or out[1] try: major, minor, _patch = line.split(" ")[1].split(".") return (int(major), int(minor)) @@ -225,7 +320,10 @@ def _resolve(name): # Absolute path: accept as-is (user knows exactly what they want). # Bare name: resolve via _which() so the override stays host-only in # Flatpak/AppImage instead of silently picking the bundled interpreter. - if os.path.isabs(override): + # In Flatpak we always defer to _which() so even absolute paths are + # checked from the host's perspective (the sandbox can't see e.g. + # /scratch/... paths that the user may have configured). + if os.path.isabs(override) and not _in_flatpak(): resolved = override if (os.path.isfile(override) and os.access(override, os.X_OK)) else "" else: diff --git a/src/testium/interpreter/utils/lua_process.py b/src/testium/interpreter/utils/lua_process.py index 5eff8f1..ec21eec 100644 --- a/src/testium/interpreter/utils/lua_process.py +++ b/src/testium/interpreter/utils/lua_process.py @@ -47,9 +47,16 @@ class LuaProcessBase: if self._process is not None: raise ETUMRuntimeError("The function subprocess has already been started.") - func_proc_path = os.path.realpath( - os.path.join(subproc_path(), "lua_func") - ) + # In Flatpak the host can't see /app/lib/testium/lua_func, so use a + # staged copy under /tmp (shared between sandbox and host). + if bins._in_flatpak(): + func_proc_path = os.path.join( + bins._get_host_testium_path(), "lua_func" + ) + else: + func_proc_path = os.path.realpath( + os.path.join(subproc_path(), "lua_func") + ) # POpen config CUST_ENV = { @@ -71,7 +78,6 @@ class LuaProcessBase: env[k] = e else: env[k] = e + ";" + env.get(k, "") - bins.apply_host_lua_paths(env) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(("localhost", 0)) @@ -79,8 +85,7 @@ class LuaProcessBase: sock.close() # POpen params - params = [ - self._lbin, + cmd_args = [ "main.lua", "--timeout", f"{self._timeout}", @@ -91,14 +96,31 @@ class LuaProcessBase: ] if tm.debug_enabled() and tm.gd("debug_rpc", False): - params.append("--verbose") + cmd_args.append("--verbose") + + if bins._in_flatpak(): + # Run on the host outside the sandbox: avoids glibc ABI mismatches + # between the Flatpak runtime and host shared libraries. + host_env = { + k: env[k] for k in ("LUA_PATH", "LUA_CPATH", "PATH") + if k in env and env[k] + } + params = bins.flatpak_host_spawn( + self._lbin, cmd_args, host_cwd=func_proc_path, + extra_env=host_env, + ) + popen_kwargs = {} + else: + params = [self._lbin, *cmd_args] + popen_kwargs = {"env": env, "cwd": func_proc_path} self._process = subprocess.Popen( - params, env=env, cwd=func_proc_path, + params, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, restore_signals=False, + **popen_kwargs, ) # Route subprocess stdout/stderr (lua require failures, syntax # errors, anything written to fd 1/2 before the in-script diff --git a/src/testium/interpreter/utils/py_process.py b/src/testium/interpreter/utils/py_process.py index 9d7367f..6ce61d6 100644 --- a/src/testium/interpreter/utils/py_process.py +++ b/src/testium/interpreter/utils/py_process.py @@ -61,14 +61,18 @@ class PyProcessBase: if sock is not None: sock.close() - # Add the path of the subprocess (root sources of testium) - tstium_path = os.path.realpath(testium_path()) - func_proc_path = os.path.realpath(subproc_path()) + # In Flatpak the host can't see /app/lib/testium, so use a staged copy + # under /tmp (shared between sandbox and host) for both cwd and as the + # root in PYTHONPATH. Outside Flatpak the original paths are used. + if bins._in_flatpak(): + tstium_path = bins._get_host_testium_path() + func_proc_path = tstium_path + else: + tstium_path = os.path.realpath(testium_path()) + func_proc_path = os.path.realpath(subproc_path()) env["PYTHONPATH"] = tstium_path + os.pathsep + self._ppath + os.pathsep + env.get("PYTHONPATH", "") - params = [ - self._pbin, - # "-m", + cmd_args = [ "py_func", "-p", f"{self._port}", @@ -77,14 +81,31 @@ class PyProcessBase: ] if tm.debug_enabled() and tm.gd("debug_rpc", False): - params.append("-v") + cmd_args.append("-v") + + if bins._in_flatpak(): + # Run on the host outside the sandbox: avoids glibc ABI mismatches + # between the Flatpak runtime and host shared libraries. + host_env = { + k: env[k] for k in ("PYTHONPATH", "PATH") + if k in env and env[k] + } + params = bins.flatpak_host_spawn( + self._pbin, cmd_args, host_cwd=func_proc_path, + extra_env=host_env, + ) + popen_kwargs = {} + else: + params = [self._pbin, *cmd_args] + popen_kwargs = {"env": env, "cwd": func_proc_path} self._process = subprocess.Popen( - params, env=env, cwd=func_proc_path, + params, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, restore_signals=False, + **popen_kwargs, ) # Route subprocess stdout/stderr (early-startup errors, # unhandled exceptions, anything written to fd 1/2 before the diff --git a/test/validation/README.md b/test/validation/README.md index 3e27e06..a5646cb 100644 --- a/test/validation/README.md +++ b/test/validation/README.md @@ -1,34 +1,67 @@ # Validation -This directory contains the testium validation suite. +This directory contains the testium validation suite. A single set of +items (`items/`), fixtures and post-processing (`post_execution.py`) is +re-used across every packaging channel. ## Running the suite ```sh -./test/validation/run.sh # Linux -test\validation\run.bat # Windows +./test/validation/run.sh # default mode = source +./test/validation/run.sh --mode wheel +./test/validation/run.sh --mode pyinstaller +./test/validation/run.sh --mode flatpak +./test/validation/run.sh --mode appimage ``` -The wrapper creates a dedicated Python venv in the system temp dir -(`${TMPDIR:-/tmp}/testium-validation-venv` on Linux, `%TEMP%\testium-validation-venv` -on Windows), using `--system-site-packages` so existing system packages -stay visible. The validation suite is then run with that venv pinned as -`python_bin`. Every test-execution subprocess (inline `<| ... |>` -evaluation, `py_func`, `cycle`, `post_execution`, ...) runs inside the -venv, while testium itself keeps running in the project's own -environment. +On Windows (only `source`, `wheel`, `pyinstaller` are supported): -Pass `clean` as the first argument to recreate the venv from scratch -(useful after a system Python upgrade): +```bat +test\validation\run.bat --mode pyinstaller +``` + +Pass `clean` as the **first** argument to recreate the validation venv +from scratch (useful after a system Python upgrade): ```sh -./test/validation/run.sh clean +./test/validation/run.sh clean --mode flatpak ``` +Any extra arguments after the mode flag are forwarded to testium. + +## Modes + +| Mode | What it launches | Prerequisite | +|---------------|-------------------------------------------------------------|------------------------------------------------------------------| +| `source` | `python3 src/testium` via the project's `run.sh` | none — works straight out of the repo | +| `wheel` | `python -m testium` inside a dedicated wheel venv | `./build_all.sh` produced `dist/testium--py3-none-any.whl` | +| `pyinstaller` | `dist/testium-` (frozen binary) | `./build_all.sh` produced the PyInstaller binary | +| `flatpak` | `flatpak run --command=testium org.testium.Testium` | the Flatpak bundle is installed (`flatpak install --user dist/testium-.flatpak`) | +| `appimage` | `dist/Testium--x86_64.AppImage` | `./build_all.sh` produced the AppImage | + +Each mode writes its results to a distinct report file +(`validation-.sqlite` / `validation--.xml`), so you +can run several modes in a row without clobbering previous reports. + +## How `python_bin` is pinned + +Every test-execution subprocess (inline `<| ... |>` evaluation, +`py_func`, `cycle`, `post_execution`, …) is routed through a dedicated +venv at `${TMPDIR:-/tmp}/testium-validation-venv`. The venv is created +with `--system-site-packages` so existing system packages stay visible, +then `junit-xml` is pip-installed for `post_execution.py`. + +This is a **host** venv. In every mode (including Flatpak) the +test-execution subprocesses end up running on the host — directly for +source/wheel/pyinstaller/appimage, and via `flatpak-spawn --host` for +Flatpak — so the same venv works across modes. The wheel mode +additionally creates a separate `testium-wheel-venv-` to hold the +installed wheel; that one is only used to launch testium itself. + ## What is checked -The `venv` item under `items/venv/` asserts that the venv is actually -being used: +The `venv` item under `items/venv/` asserts that the validation venv is +actually being used: * `python_bin` is set in the global dict. * The eval subprocess (used for `<| ... |>` expressions) has diff --git a/test/validation/run.bat b/test/validation/run.bat index b481b43..f540504 100644 --- a/test/validation/run.bat +++ b/test/validation/run.bat @@ -1,61 +1,131 @@ @echo off -SETLOCAL EnableExtensions +SETLOCAL EnableExtensions EnableDelayedExpansion -REM Runs the testium validation suite with a dedicated Python venv used -REM by every py_func / cycle / inline-eval subprocess. testium itself -REM keeps running in the project's own environment; the validation venv -REM only isolates *test execution*. +REM Runs the testium validation suite against any installable channel of +REM testium on Windows (source, wheel, pyinstaller). REM -REM test\validation\run.bat [clean] [extra testium args] +REM Usage: +REM test\validation\run.bat [clean] [--mode MODE] [extra testium args] REM -REM Requires the project venv to already exist (run the project's -REM run.bat once first, or any other testium install method). +REM clean remove the validation venv before recreating it +REM (must be the first argument; useful after a Python upgrade) +REM +REM --mode MODE which testium build to validate. One of: +REM source (default) project's run.bat (src\testium) +REM wheel dist\testium--py3-none-any.whl +REM pyinstaller dist\testium-.exe (or dist\testium-) +REM +REM Every test-execution subprocess runs in a dedicated host venv under +REM %TEMP%\testium-validation-venv (created with --system-site-packages, +REM then junit-xml is pip-installed for post_execution.py). +REM +REM The report file is suffixed with the mode so consecutive runs in +REM different modes don't overwrite each other. SET "SCRIPT_DIR=%~dp0" -SET "PROJECT_DIR=%SCRIPT_DIR%..\.." -REM Venv in the user temp dir (Windows equivalent of /tmp). -SET "VENV_DIR=%TEMP%\testium-validation-venv" -SET "PROJECT_VENV=%PROJECT_DIR%\test\tmp\testium_venv" +IF "%SCRIPT_DIR:~-1%"=="\" SET "SCRIPT_DIR=%SCRIPT_DIR:~0,-1%" +SET "PROJECT_DIR=%SCRIPT_DIR%\..\.." +SET /P VERSION=<"%PROJECT_DIR%\src\VERSION" +REM ---------- arg parsing ---------------------------------------------------- + +SET "MODE=source" +SET "CLEAN=0" IF /I "%~1"=="clean" ( - rmdir /s /q "%VENV_DIR%" + SET "CLEAN=1" SHIFT ) -REM Locate a host Python. -SET "PYTHON_EXE=python" +SET "EXTRA=" +:PARSE_ARGS +IF "%~1"=="" GOTO ARGS_DONE +IF /I "%~1"=="--mode" ( + SET "MODE=%~2" + SHIFT + SHIFT + GOTO PARSE_ARGS +) +SET "EXTRA=!EXTRA! "%~1"" +SHIFT +GOTO PARSE_ARGS +:ARGS_DONE + +REM ---------- locate host python --------------------------------------------- + +SET "PYTHON_EXE=" py --version >nul 2>&1 IF %ERRORLEVEL% EQU 0 ( SET "PYTHON_EXE=py" - goto :PYTHON_FOUND + GOTO PYTHON_FOUND ) python --version >nul 2>&1 IF %ERRORLEVEL% EQU 0 ( SET "PYTHON_EXE=python" - goto :PYTHON_FOUND + GOTO PYTHON_FOUND ) -echo ERROR : Python could not be found on this system. +echo ERROR: Python could not be found on this system. exit /b 1 - :PYTHON_FOUND +REM ---------- validation venv ------------------------------------------------- + +SET "VENV_DIR=%TEMP%\testium-validation-venv" +IF "%CLEAN%"=="1" IF EXIST "%VENV_DIR%" rmdir /s /q "%VENV_DIR%" + IF NOT EXIST "%VENV_DIR%" ( echo Creating validation venv at %VENV_DIR% %PYTHON_EXE% -m venv --system-site-packages "%VENV_DIR%" - IF %ERRORLEVEL% NEQ 0 ( + IF !ERRORLEVEL! NEQ 0 ( echo ERROR while creating the validation venv. exit /b 1 ) call "%VENV_DIR%\Scripts\pip" install --quiet --upgrade pip call "%VENV_DIR%\Scripts\pip" install --quiet junit-xml ) - SET "VENV_PYTHON=%VENV_DIR%\Scripts\python.exe" -IF NOT EXIST "%PROJECT_VENV%" ( - echo ERROR : project venv not found at %PROJECT_VENV%. Run the project run.bat once first. +REM ---------- shared "tail" forwarded to every launcher ----------------------- +REM Reports are stamped with the mode so successive runs don't clobber each other. + +SET "TAIL=-b -d "python_bin=%VENV_PYTHON%" -d "validation_report_file=validation-%MODE%" -- "%SCRIPT_DIR%\main.tum"%EXTRA%" + +REM ---------- per-mode launcher ---------------------------------------------- + +echo -- validation mode: %MODE% + +IF /I "%MODE%"=="source" GOTO MODE_SOURCE +IF /I "%MODE%"=="wheel" GOTO MODE_WHEEL +IF /I "%MODE%"=="pyinstaller" GOTO MODE_PYI +echo ERROR: unknown --mode '%MODE%'. Expected: source ^| wheel ^| pyinstaller. +exit /b 1 + +:MODE_SOURCE +call "%PROJECT_DIR%\run.bat" %TAIL% +exit /b %ERRORLEVEL% + +:MODE_WHEEL +SET "WHEEL=%PROJECT_DIR%\dist\testium-%VERSION%-py3-none-any.whl" +IF NOT EXIST "%WHEEL%" ( + echo ERROR: wheel not found at %WHEEL% -- run build_all.sh first. exit /b 1 ) +SET "WHEEL_VENV=%TEMP%\testium-wheel-venv-%VERSION%" +IF "%CLEAN%"=="1" IF EXIST "%WHEEL_VENV%" rmdir /s /q "%WHEEL_VENV%" +IF NOT EXIST "%WHEEL_VENV%" ( + echo Creating wheel venv at %WHEEL_VENV% + %PYTHON_EXE% -m venv --system-site-packages "%WHEEL_VENV%" + call "%WHEEL_VENV%\Scripts\pip" install --quiet --upgrade pip + call "%WHEEL_VENV%\Scripts\pip" install --quiet "%WHEEL%" +) +"%WHEEL_VENV%\Scripts\python.exe" -m testium %TAIL% +exit /b %ERRORLEVEL% -call "%PROJECT_VENV%\Scripts\activate" -python "%PROJECT_DIR%\src\testium" -b -d "python_bin=%VENV_PYTHON%" -- "%SCRIPT_DIR%main.tum" %* +:MODE_PYI +SET "PYI_BIN=%PROJECT_DIR%\dist\testium-%VERSION%.exe" +IF NOT EXIST "%PYI_BIN%" SET "PYI_BIN=%PROJECT_DIR%\dist\testium-%VERSION%" +IF NOT EXIST "%PYI_BIN%" ( + echo ERROR: PyInstaller binary not found in %PROJECT_DIR%\dist -- run build_all.sh first. + exit /b 1 +) +"%PYI_BIN%" %TAIL% +exit /b %ERRORLEVEL% diff --git a/test/validation/run.sh b/test/validation/run.sh index c86bc02..5a1b083 100755 --- a/test/validation/run.sh +++ b/test/validation/run.sh @@ -1,47 +1,143 @@ #!/bin/bash -# Runs the testium validation suite with a dedicated Python venv used by -# every py_func / cycle / inline-eval subprocess (i.e. everything that -# goes through ``bins.python_bin()``). testium itself keeps running in -# the project's own environment — the validation venv only isolates -# *test execution*. +# Runs the testium validation suite against any installable channel of +# testium (source, wheel, pyinstaller, flatpak, appimage). # -# ./test/validation/run.sh [clean] [extra testium args] +# Usage: +# ./test/validation/run.sh [clean] [--mode MODE] [extra testium args] # -# ``clean`` (optional, must be the first arg) removes the venv before -# recreating it; this is the way to refresh the venv after a system -# Python upgrade. +# clean remove the validation venv before recreating it +# (must be the first argument; useful after a Python upgrade) +# +# --mode MODE which testium build to validate. One of: +# source (default) src/testium via project run.sh +# wheel dist/testium--py3-none-any.whl +# pyinstaller dist/testium- +# flatpak installed org.testium.Testium +# appimage dist/Testium--*.AppImage +# +# Every test-execution subprocess (inline <| ... |>, py_func, cycle, +# post_execution, ...) runs in a dedicated host venv under +# /tmp/testium-validation-venv. That venv is shared across modes — +# even Flatpak reaches it via flatpak-spawn --host. The validation venv +# is created with --system-site-packages so existing system packages +# (PySide6, lxml, ...) stay visible, then junit-xml is pip-installed +# for post_execution.py. +# +# The report file is suffixed with the mode (e.g. validation-flatpak.sqlite) +# so consecutive runs in different modes don't overwrite each other. set -e SCRIPT_PATH="$(readlink -f "$0")" SCRIPT_DIR="$(realpath "$(dirname "$SCRIPT_PATH")")" PROJECT_DIR="$(realpath "$SCRIPT_DIR/../..")" -# Venv lives in the system temp dir so it stays out of the project tree -# (and is naturally cleaned up by tmpfiles/reboot on most distros). -VENV_DIR="${TMPDIR:-/tmp}/testium-validation-venv" +VERSION="$(cat "$PROJECT_DIR/src/VERSION")" + +# ---------- arg parsing ------------------------------------------------------- + +MODE=source if [ "${1:-}" = "clean" ]; then - rm -rf "$VENV_DIR" + CLEAN=1 shift +else + CLEAN=0 +fi + +EXTRA=() +while [ $# -gt 0 ]; do + case "$1" in + --mode) + MODE="$2" + shift 2 + ;; + --mode=*) + MODE="${1#--mode=}" + shift + ;; + *) + EXTRA+=("$1") + shift + ;; + esac +done + +# ---------- validation venv --------------------------------------------------- + +VENV_DIR="${TMPDIR:-/tmp}/testium-validation-venv" +if [ "$CLEAN" -eq 1 ]; then + rm -rf "$VENV_DIR" fi if [ ! -d "$VENV_DIR" ]; then echo "Creating validation venv at $VENV_DIR" - # --system-site-packages so we don't have to reinstall pyside6, lxml - # & friends just to support the validation helpers. We still pip - # install junit-xml below because it is the one dep that does *not* - # ship as a system package on most distros and is required by - # post_execution.py. python3 -m venv --system-site-packages "$VENV_DIR" "$VENV_DIR/bin/pip" install --quiet --upgrade pip "$VENV_DIR/bin/pip" install --quiet junit-xml fi - VENV_PYTHON="$VENV_DIR/bin/python3" -# Delegate to the project's run.sh so testium itself still runs in the -# project venv (with pyside6, gitpython, ...). ``-d python_bin=...`` -# pins every test-execution subprocess to the validation venv. -exec "$PROJECT_DIR/run.sh" -b \ +# ---------- per-mode launcher ------------------------------------------------- + +case "$MODE" in + source) + CMD=("$PROJECT_DIR/run.sh") + ;; + wheel) + WHEEL="$PROJECT_DIR/dist/testium-${VERSION}-py3-none-any.whl" + if [ ! -f "$WHEEL" ]; then + echo "ERROR: wheel not found at $WHEEL — run ./build_all.sh first." >&2 + exit 1 + fi + WHEEL_VENV="${TMPDIR:-/tmp}/testium-wheel-venv-${VERSION}" + if [ "$CLEAN" -eq 1 ]; then + rm -rf "$WHEEL_VENV" + fi + if [ ! -d "$WHEEL_VENV" ]; then + echo "Creating wheel venv at $WHEEL_VENV" + python3 -m venv --system-site-packages "$WHEEL_VENV" + "$WHEEL_VENV/bin/pip" install --quiet --upgrade pip + "$WHEEL_VENV/bin/pip" install --quiet "$WHEEL" + fi + CMD=("$WHEEL_VENV/bin/python" -m testium) + ;; + pyinstaller) + PYI_BIN="$PROJECT_DIR/dist/testium-${VERSION}" + if [ ! -x "$PYI_BIN" ]; then + echo "ERROR: PyInstaller binary not found at $PYI_BIN — run ./build_all.sh first." >&2 + exit 1 + fi + CMD=("$PYI_BIN") + ;; + flatpak) + if ! flatpak info --user org.testium.Testium &>/dev/null \ + && ! flatpak info --system org.testium.Testium &>/dev/null; then + echo "ERROR: org.testium.Testium is not installed." >&2 + echo " flatpak install --user $PROJECT_DIR/dist/testium-${VERSION}.flatpak" >&2 + exit 1 + fi + CMD=(flatpak run --command=testium org.testium.Testium) + ;; + appimage) + APPIMAGE=$(ls -1t "$PROJECT_DIR/dist"/Testium-"${VERSION}"-*.AppImage 2>/dev/null | head -1) + if [ -z "$APPIMAGE" ] || [ ! -x "$APPIMAGE" ]; then + echo "ERROR: no AppImage for version $VERSION under $PROJECT_DIR/dist — run ./build_all.sh first." >&2 + exit 1 + fi + CMD=("$APPIMAGE") + ;; + *) + echo "ERROR: unknown --mode '$MODE'. Expected: source|wheel|pyinstaller|flatpak|appimage." >&2 + exit 1 + ;; +esac + +# ---------- launch ------------------------------------------------------------ + +echo "-- validation mode: $MODE" +echo "-- launch: ${CMD[*]}" + +exec "${CMD[@]}" -b \ -d "python_bin=$VENV_PYTHON" \ - -- "$SCRIPT_DIR/main.tum" "$@" + -d "validation_report_file=validation-$MODE" \ + -- "$SCRIPT_DIR/main.tum" "${EXTRA[@]}"