François
7abd8c07a6
load_errors_check.py loads deliberately broken .tum fixtures in batch on the build under test (like lsp_check.py) and asserts each fails with its specific located message and without a raw traceback. Wired into run.sh just before the main suite, so it runs for every channel. The run validation items now point their sub-instance log at the gitignored report dir, so a GUI run no longer litters the tree with sub_*.log files. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
88 lines
3.5 KiB
Python
88 lines
3.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Per-channel check of test-load error reporting.
|
|
|
|
Given the channel's testium invocation as argv (e.g. ``flatpak run
|
|
--command=testium org.testium.Testium``, a PyInstaller binary path, or
|
|
``python -m testium``), load each deliberately broken ``.tum`` under
|
|
``load_errors/`` in batch mode and verify that:
|
|
|
|
1. the load FAILS (non-zero exit), and
|
|
2. the output carries the *specific, located* message we expect — not a bare
|
|
Python traceback and not the generic 'crashed for any reason'.
|
|
|
|
This guards the load-time error handling in ``test_set.load_test_recursively``
|
|
and ``item_actions.load`` (a structural mistake in a ``.tum`` must always reach
|
|
the user as a readable ``TUM file syntax error`` naming the offending file,
|
|
item path and value). The historical failure mode was an unknown console
|
|
action crashing the error formatter itself with ``'dict_keys' object is not
|
|
subscriptable``.
|
|
|
|
Exits non-zero (with a diagnostic) on the first failure so the validation run
|
|
fails loudly. Used by ``run.sh`` before launching the main suite.
|
|
"""
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
FIXTURES = os.path.join(HERE, "load_errors")
|
|
|
|
# testium colourises its log; strip the ANSI escapes before matching messages.
|
|
_ANSI = re.compile(r"\x1b\[[0-9;]*m")
|
|
|
|
# fixture file -> substrings that must all appear in the load output.
|
|
CASES = [
|
|
("unknown_item.tum", ["TUM file syntax error", "is not a known test item",
|
|
"frobnicate", "Known items:"]),
|
|
("unknown_action.tum", ["unknown action", "opens", "Known actions:"]),
|
|
("two_steps.tum", ["must define exactly one test item"]),
|
|
("scalar_body.tum", ["body of test item 'sleep'", "must be a mapping"]),
|
|
("group_no_steps.tum", ["No 'steps' list found", "'group' item 'g'"]),
|
|
("step_not_mapping.tum", ["is not a valid test item"]),
|
|
# The error is inside the included file: the message must name that file.
|
|
("bad_include.tum", ["bad_include_inc.tum", "frobnicate_in_include",
|
|
"is not a known test item"]),
|
|
]
|
|
|
|
|
|
def fail(msg):
|
|
print(f"LOAD-ERROR CHECK: FAIL — {msg}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
def check_case(cmd, fixture, needles):
|
|
path = os.path.join(FIXTURES, fixture)
|
|
try:
|
|
out = subprocess.run(cmd + ["-b", path], capture_output=True, timeout=120)
|
|
except Exception as e: # noqa: BLE001
|
|
fail(f"`{' '.join(cmd)} -b {fixture}` could not run: {e}")
|
|
blob = _ANSI.sub("", (out.stdout + out.stderr).decode(errors="replace"))
|
|
|
|
if out.returncode == 0 or "Test run success." in blob:
|
|
fail(f"{fixture}: load was expected to fail but succeeded "
|
|
f"(exit {out.returncode}).")
|
|
# A raw Python traceback reaching the user is exactly what we are guarding
|
|
# against: every load error must be funnelled through a TUM*Error.
|
|
if "Traceback (most recent call last)" in blob:
|
|
fail(f"{fixture}: a raw Python traceback leaked to the user:\n"
|
|
f"{blob[-600:]}")
|
|
missing = [n for n in needles if n not in blob]
|
|
if missing:
|
|
fail(f"{fixture}: load message is missing {missing}.\n"
|
|
f"--- got ---\n{blob[-800:]}")
|
|
print(f"LOAD-ERROR CHECK: {fixture} OK")
|
|
|
|
|
|
def main():
|
|
cmd = sys.argv[1:]
|
|
if not cmd:
|
|
fail("usage: load_errors_check.py <testium-invocation...>")
|
|
for fixture, needles in CASES:
|
|
check_case(cmd, fixture, needles)
|
|
print("LOAD-ERROR CHECK: PASS")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|