release preparation

Set the app id via setDesktopFileName so the window stops inheriting the
launcher's class ("python3" under the AppImage), which is what GNOME was
keying the wrong icon off. On native Wayland the task-bar icon comes from
an installed desktop file matched to the app id (setWindowIcon is
ignored there), so on Linux drop an idempotent desktop entry + 256px icon
under ~/.local/share. Flatpak keeps its own id/desktop; Windows / macOS
use the window icon. No-op off Linux.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

release_note.txt

@@ -1,3 +1,16 @@
+version 0.2.1
+==============
+- Faster test loading, especially for large tests built from jinja
+  templates and ``!include``: compiled jinja templates are cached and
+  reused (a file included many times is compiled once), rendering happens
+  in memory instead of through a temporary file, and YAML is parsed with
+  the libyaml C loader when available. Typical load time is 3-6x lower on
+  include / template-heavy tests; behaviour is unchanged.
+- Fix: a nested list holding more than one step under ``steps`` no longer
+  duplicates its entries while the step tree is built.
+- New load-time benchmark under ``test/benchmark/`` (synthetic-tree
+  generator + in-process timing harness) to measure the load pipeline.
+
 version 0.2
 ==============
 - Test items: each item type now declares its accepted parameters

src/VERSION

@@ -1 +1 @@
-0.2
+0.2.1

src/testium/interpreter/test_set.py

@@ -29,6 +29,51 @@ def _build_item_path(item) -> str:
     return " > ".join(reversed(parts))
 
 
+def _flatten_actions(actions, out, parent_seq_name):
+    """Expand nested lists and included ``sequence`` entries into ``out`` as a
+    flat list of single test-item dicts, propagating each sequence's source
+    filename onto its items.
+
+    Replaces the previous approach, which spliced each entry into the step
+    list and rebuilt the whole list every time (O(n^2) over the step list, and
+    a rebuild that duplicated entries when a nested list held more than one
+    element). This single forward pass is linear.
+    """
+    for idx, action in enumerate(actions):
+        # a bare list raises its elements to the same level
+        if isinstance(action, (list, tuple)):
+            _flatten_actions(action, out, parent_seq_name)
+            continue
+        # a NoneType (e.g. pointing at an unused alias) contributes nothing
+        if action is None:
+            continue
+        # a 'sequence' (an included file) is spliced in, with its filename
+        # propagated onto each of its items
+        if isinstance(action, dict) and "sequence" in action:
+            sequence = action["sequence"]["data"]
+            f = action["sequence"]["filename"]
+            if isinstance(sequence, dict):
+                sequence = [{k: v} for k, v in sequence.items()]
+            # Case of an empty sequence
+            elif sequence is None:
+                tm.print_info(
+                    f"An empty sequence is loaded in '{parent_seq_name}'."
+                )
+                sequence = []
+            elif not isinstance(sequence, list):
+                raise ETUMSyntaxError(
+                    f"Syntax error in '{parent_seq_name}' step number {idx+1}. Sequence definition: '{str(action)}'",
+                    f
+                )
+            for s in sequence:
+                if isinstance(s, dict) and s:
+                    s[list(s.keys())[0]]["seq_filename"] = f
+            _flatten_actions(sequence, out, parent_seq_name)
+            continue
+
+        out.append(action)
+
+
 class TestSet:
     def __init__(
         self,
@@ -434,56 +479,16 @@ class TestSet:
                 f"No valid list of actions in sequence {parent_seq_name}",
                 file_name
             )
-        # first we merged to the same level 'sequence dict entries and list within the list
-        counter = 0
         test_dir = tm.gd("test_directory")
-        la = len(parent_seq_actions)
-        while counter < la:
-            action = parent_seq_actions[counter]
-            # if action is a list raise up to the the same level,
-            # ie insert action element into the parent_seq_actions
-            if isinstance(action, (list, tuple)):
-                parent_seq_actions[counter : counter + 1] = action
-                parent_seq_actions = (
-                    parent_seq_actions[:counter]
-                    + action
-                    + parent_seq_actions[counter + 1 :]
-                )
-                la = len(parent_seq_actions)
-                continue
-            # if action is a NoneType skip and continue
-            # (when pointing to an unused alias for instance)
-            if action is None:
-                counter += 1
-                continue
-            # if action is a sequence we insert its entry into the action list
-            if "sequence" in action:
-                sequence = action["sequence"]["data"]
-                f = action["sequence"]["filename"]
-                if isinstance(sequence, dict):
-                    sequence = [{k: v} for k, v in sequence.items()]
-                # Case of an empty sequence
-                elif sequence is None:
-                    tm.print_info(
-                        f"An empty sequence is loaded in '{parent_seq_name}'."
-                    )
-                    sequence = []
-                elif not isinstance(sequence, list):
-                    raise ETUMSyntaxError(
-                        f"Syntax error in '{parent_seq_name}' step number {counter+1}. Sequence definition: '{str(action)}'",
-                        f
-                    )
-                for s in sequence:
-                    s[list(s.keys())[0]]["seq_filename"] = f
-                parent_seq_actions = (
-                    parent_seq_actions[:counter]
-                    + sequence
-                    + parent_seq_actions[counter + 1 :]
-                )
-                la = len(parent_seq_actions)
-                continue
 
-            # Action is now for sure a list of dict of length 1
+        # Flatten nested lists and included 'sequence' entries to the same level
+        # in one linear pass (was an in-place splice + full list rebuild per
+        # entry: O(n^2) over the step list).
+        flat_actions = []
+        _flatten_actions(parent_seq_actions, flat_actions, parent_seq_name)
+
+        for action in flat_actions:
+            # Action is now for sure a dict of length 1
             k = list(action.keys())[0]
             if action[k].get("seq_filename", None) is None:
                 action[k]["seq_filename"] = file_name
@@ -546,8 +551,6 @@ class TestSet:
                     action[k]["seq_filename"]
                 )
 
-            counter += 1
-
         return ret
 
     def tree(self):

src/testium/interpreter/utils/include.py

@@ -6,10 +6,10 @@ from runtime.tum_except import ETUMFileError
 from interpreter.utils.template import template_to_test
 from copy import copy
 from interpreter.utils.globdict import global_dict
-from interpreter.utils.yaml_load import yaml_load
+from interpreter.utils.yaml_load import yaml_load, YAML_BASE_LOADER
 
 
-class TUMLoaderNoIncludes(yaml.Loader):
+class TUMLoaderNoIncludes(YAML_BASE_LOADER):
 
     def __init__(self, stream):
 

src/testium/interpreter/utils/template.py

@@ -1,33 +1,74 @@
+import io
 import os
 from sys import exc_info
-from jinja2 import Template
+from jinja2 import Environment
 from jinja2.exceptions import TemplateSyntaxError, TemplateError, UndefinedError
-from tempfile import TemporaryFile
 from interpreter.utils.yaml_load import print_yaml
 from runtime.tum_except import ETUMSyntaxError
 
 
+# One Environment reused for every render (default settings, i.e. identical
+# behaviour to jinja2.Template), plus a compiled-template cache so a file that
+# is included many times — or a test that is reloaded — is compiled only once.
+# Jinja compilation is the expensive step; render (variable substitution) stays
+# per-call. Cache is keyed on path + mtime + size so an edited file recompiles.
+_ENV = Environment()
+_template_cache = {}  # abspath -> (mtime_ns, size, compiled_template)
+
+
+class _RenderedStream(io.StringIO):
+    """A rendered template kept in memory.
+
+    Carries ``root`` (and ``name``) so the YAML loader resolves ``!include``
+    paths exactly as it did from the on-disk temp file this replaces — without
+    the write + seek + read round-trip (one temp file per included file). That
+    round-trip is pure overhead, and especially costly on slow storage.
+    """
+
+
+def _compiled_template(filename: str):
+    """Return the compiled jinja template for *filename*, reusing the cached
+    one when the file is unchanged (path + mtime + size)."""
+    key = os.path.abspath(filename)
+    try:
+        st = os.stat(filename)
+    except OSError:
+        st = None
+    if st is not None:
+        cached = _template_cache.get(key)
+        if (cached is not None
+                and cached[0] == st.st_mtime_ns
+                and cached[1] == st.st_size):
+            return cached[2]
+    with open(filename, "r") as f:
+        source = f.read()
+    template = _ENV.from_string(source)  # compile (may raise TemplateSyntaxError)
+    if st is not None:
+        _template_cache[key] = (st.st_mtime_ns, st.st_size, template)
+    return template
+
+
 def template_to_test(filename: str, params: list):
     """ Function which processes an eventual jinja2 template to a test file
     """
-    # Temporary file created to receive the processed include
-    # file
-    tmpf = TemporaryFile('w+t')
-    with open(filename, 'r') as f:
-        try:
-            j2_template = Template(f.read())
-        except TemplateError as e:
+    # Compile (cached) — a syntax error in the template surfaces here.
+    try:
+        j2_template = _compiled_template(filename)
+    except TemplateError as e:
+        with open(filename, "r") as f:
             print_yaml(f, filename)
-            type, value, tb = exc_info()
-            msg = "Template error"
-            if hasattr(value, 'lineno'):
-                msg = msg + f" on line {value.lineno}: "
-            else:
-                msg += ": "
-            raise ETUMSyntaxError(msg + str(e), filename)
+        type, value, tb = exc_info()
+        msg = "Template error"
+        if hasattr(value, 'lineno'):
+            msg = msg + f" on line {value.lineno}: "
+        else:
+            msg += ": "
+        raise ETUMSyntaxError(msg + str(e), filename)
+
+    # Render into memory (no temp file).
     try:
         params["include_directory"] = os.path.dirname(os.path.abspath(filename))
-        tmpf.write(j2_template.render(params))
+        rendered = j2_template.render(params)
     except TemplateSyntaxError as e:
         raise ETUMSyntaxError(f"""Template loading of file '{filename}' with following parameters '{str(params)}'
 Syntax error in template: {e.message}""")
@@ -42,8 +83,7 @@ Template rendering error: {e.message}""")
         raise ETUMSyntaxError(f"""Template loading of file '{filename}' with following parameters '{str(params)}'
 Unexpected error: {str(e)}""")
 
-    # return to begining of the temp file
-    tmpf.seek(0, os.SEEK_SET)
-    tmpf.root = os.path.dirname(filename)
-
-    return tmpf
+    stream = _RenderedStream(rendered)
+    stream.root = os.path.dirname(filename)
+    stream.name = filename
+    return stream

src/testium/interpreter/utils/test_init.py

@@ -11,7 +11,7 @@ import api.testium as tm
 import interpreter.utils.globdict as globdict
 import interpreter.utils.settings as prefs
 from interpreter.utils.paths import testium_path
-from interpreter.utils.yaml_load import yaml_load
+from interpreter.utils.yaml_load import yaml_load, YAML_BASE_LOADER
 from interpreter.utils import clear_recursively
 from runtime.tum_except import ETUMSyntaxError
 from interpreter.utils.params import expanse, eval_func_init
@@ -89,7 +89,7 @@ def locate_report_file(rep_file):
 def yamltodict(param_file, silent=True):
     # load of the file
     with open(param_file, "r") as fd:
-        dp = yaml_load(fd, param_file, yaml.Loader)
+        dp = yaml_load(fd, param_file, YAML_BASE_LOADER)
 
     if dp is None:
         tm.print_info(f"The YAML file '{param_file}' is empty.")

src/testium/interpreter/utils/yaml_load.py

@@ -1,3 +1,4 @@
+import yaml
 from yaml.parser import ParserError
 from yaml import load, Loader
 from yaml.scanner import ScannerError
@@ -5,6 +6,12 @@ from api.testium import print_debug
 from runtime.tum_except import ETUMSyntaxError
 import io
 
+# Use the libyaml-backed loader (much faster parsing) when PyYAML was built
+# with it, falling back to the pure-Python loader otherwise. The C loader
+# raises the same ParserError/ScannerError and supports the same custom
+# constructors (!include) and construct_* helpers the TUM loaders rely on.
+YAML_BASE_LOADER = yaml.CLoader if getattr(yaml, "__with_libyaml__", False) else yaml.Loader
+
 
 def print_yaml(file: io.TextIOWrapper, file_name):
     """ Prints YAML file if debug mode is activated.
@@ -21,10 +28,10 @@ def yaml_load(file, real_file_name: str, loader: Loader):
         return load(file, loader)
 
     except ParserError as e:
-        if isinstance(file, io.TextIOWrapper):
+        if isinstance(file, (io.TextIOWrapper, io.StringIO)):
             print_yaml(file, real_file_name)
         raise ETUMSyntaxError(f"yaml file parsing error: " + str(e), real_file_name)
     except ScannerError as e:
-        if isinstance(file, io.TextIOWrapper):
+        if isinstance(file, (io.TextIOWrapper, io.StringIO)):
             print_yaml(file, real_file_name)
         raise ETUMSyntaxError("yaml file scanning error: " + str(e), real_file_name)

src/testium/main_win/desktop_integration.py

@@ -0,0 +1,95 @@
+"""Install a desktop entry + icon under the user's data dir so desktop shells
+show the testium icon in the task bar / dock.
+
+On a native Wayland session GNOME takes a window's task-bar icon from the
+``.desktop`` file whose name (or ``StartupWMClass``) matches the window
+``app_id`` — ``QGuiApplication.setWindowIcon`` is ignored there. The portable
+channels (source checkout, PyInstaller binary, AppImage) install no system
+desktop file, so we drop an idempotent one in ``~/.local/share``. The window
+``app_id`` is set to ``testium`` (see ``QApplication.setDesktopFileName`` in
+``testium_win``), which is exactly this file's base name.
+
+Flatpak ships its own ``org.testium.Testium.desktop`` and keeps its own app id,
+so the caller skips this integration there.
+"""
+import os
+import sys
+
+from PySide6.QtCore import Qt
+from PySide6.QtGui import QPixmap
+
+# Must match QApplication.setDesktopFileName(...) for the GUI, and is used as
+# both the desktop-file base name and the StartupWMClass.
+APP_ID = "testium"
+
+
+def _launch_command():
+    """Best-effort Exec= for the menu entry. Not needed for icon matching, but
+    makes the entry actually launchable when possible."""
+    appimage = os.environ.get("APPIMAGE")
+    if appimage:
+        return f'"{appimage}"'
+    if getattr(sys, "frozen", False):
+        return f'"{os.path.abspath(sys.executable)}"'
+    argv0 = os.path.abspath(sys.argv[0]) if sys.argv and sys.argv[0] else ""
+    if argv0 and os.path.exists(argv0):
+        return f'"{os.path.abspath(sys.executable)}" "{argv0}"'
+    return f'"{os.path.abspath(sys.executable)}" -m testium'
+
+
+def ensure_desktop_entry():
+    """Create (or refresh) ~/.local/share icon + desktop entry. Best-effort:
+    any failure is swallowed so it can never take the GUI down.
+
+    Freedesktop-only: a no-op off Linux (Windows / macOS use the window icon)."""
+    if not sys.platform.startswith("linux"):
+        return
+    try:
+        data_home = os.environ.get("XDG_DATA_HOME") or os.path.join(
+            os.path.expanduser("~"), ".local", "share"
+        )
+        icon_dir = os.path.join(data_home, "icons", "hicolor", "256x256", "apps")
+        app_dir = os.path.join(data_home, "applications")
+        icon_path = os.path.join(icon_dir, f"{APP_ID}.png")
+        desktop_path = os.path.join(app_dir, f"{APP_ID}.desktop")
+
+        os.makedirs(icon_dir, exist_ok=True)
+        os.makedirs(app_dir, exist_ok=True)
+
+        # Icon: render the bundled Qt resource to a PNG once. Requires a live
+        # QGuiApplication (the caller creates it before calling us).
+        if not os.path.isfile(icon_path):
+            pixmap = QPixmap(u":/black/testium_logo.png")
+            if not pixmap.isNull():
+                pixmap = pixmap.scaled(
+                    256, 256,
+                    Qt.AspectRatioMode.KeepAspectRatio,
+                    Qt.TransformationMode.SmoothTransformation,
+                )
+                pixmap.save(icon_path, "PNG")
+
+        # Absolute Icon= path so the shell resolves it without an icon-cache
+        # refresh; StartupWMClass lets X11 / XWayland match too.
+        desktop = (
+            "[Desktop Entry]\n"
+            "Type=Application\n"
+            "Name=Testium\n"
+            "Comment=Test sequencer\n"
+            f"Icon={icon_path}\n"
+            f"Exec={_launch_command()} %f\n"
+            "Terminal=false\n"
+            f"StartupWMClass={APP_ID}\n"
+            "Categories=Utility;Development;\n"
+        )
+
+        # Write only when missing or changed, to avoid needless menu churn.
+        current = None
+        if os.path.isfile(desktop_path):
+            with open(desktop_path, "r") as fh:
+                current = fh.read()
+        if current != desktop:
+            with open(desktop_path, "w") as fh:
+                fh.write(desktop)
+    except Exception:
+        # Desktop integration is a nicety, never a hard requirement.
+        pass

src/testium/main_win/testium_win.py

@@ -678,6 +678,24 @@ def MainWin(
     debug=False,
 ):
     app = QApplication(sys.argv)
+    # Application identity so desktop shells (GNOME, ...) show the testium
+    # icon in the task bar / dock instead of a generic one. On Wayland this
+    # sets the surface app_id; on X11/XWayland it sets WM_CLASS, so the window
+    # stops inheriting the launcher's class (e.g. "python3" under the AppImage,
+    # which is what GNOME was keying the wrong icon off) and the window icon
+    # below is used as the fallback. In Flatpak the id must be the Flatpak app
+    # id so it matches the installed desktop file.
+    app.setApplicationName("Testium")
+    app.setApplicationDisplayName("Testium")
+    app.setDesktopFileName(os.environ.get("FLATPAK_ID", "testium"))
+    app.setWindowIcon(QIcon(u":/black/testium_logo.png"))
+    # On native Wayland the task-bar icon comes from an installed desktop file
+    # matched to the app_id, not from setWindowIcon(). Flatpak ships its own;
+    # for the other Linux channels drop an idempotent one under ~/.local/share.
+    # Windows / macOS use the window icon set above, so this is Linux-only.
+    if sys.platform.startswith("linux") and not os.environ.get("FLATPAK_ID"):
+        from main_win.desktop_integration import ensure_desktop_entry
+        ensure_desktop_entry()
     ui = MainWindow(
         test_file,
         config_files,

test/benchmark/.gitignore

@@ -0,0 +1 @@
+cases/

test/benchmark/README.md

@@ -0,0 +1,116 @@
+# Load-time benchmark
+
+Measures how long *testium* takes to **load** a `.tum` test tree — template
+rendering (jinja) + YAML parsing + test-tree construction — *without* executing
+it. Purpose: get reproducible numbers before/after load-path optimisations, and
+attribute any gain to a specific part of the pipeline.
+
+It is meant for *very long* tests, the kind you can build with `jinja` loops and
+`!include`, where load time becomes noticeable.
+
+## Files
+
+| File | Role |
+|------|------|
+| `gen_bench_test.py` | Generates a synthetic `.tum` tree (the test input). |
+| `load_bench.py` | Drives the **real** loader in-process and times it. |
+| `run.sh` | Convenience: generate + time across profiles, using the project venv. |
+| `cases/` | Generated trees (git-ignored, recreated on demand). |
+
+The benchmark `.tum` files are **generated**, not committed — the generator is
+the artifact. They use only `let` leaves and `group` containers, so loading has
+no runtime side effect (no subprocess, no `<| |>` eval) and the timing reflects
+the parse/build pipeline alone.
+
+## Quick start
+
+```bash
+# default matrix (all profiles), 5 repeats each
+./test/benchmark/run.sh
+
+# one profile at one size
+./test/benchmark/run.sh repeat 2000
+
+# more repeats for a tighter min
+REPEAT=10 ./test/benchmark/run.sh includes 1000
+```
+
+`run.sh` uses the project venv at `test/tmp/.venv` (created by `./run.sh`). If it
+is missing, run `./run.sh` once first.
+
+To drive the harness directly on any `.tum` (not just generated ones):
+
+```bash
+test/tmp/.venv/bin/python3 test/benchmark/load_bench.py --repeat 5 --quiet path/to/main.tum
+```
+
+## Profiles
+
+Each profile isolates one cost. `--size` is the profile-specific count.
+
+| Profile | What it builds | Stresses |
+|---------|----------------|----------|
+| `flat` | one main file, N inline `let` steps | big YAML parse + linear object build |
+| `includes` | main `!include`s N **distinct** sub-files | per-include template+YAML+tempfile, `sequence` splice |
+| `repeat` | main `!include`s the **same** parametrised leaf N times | jinja **recompilation** of an identical template |
+| `jinja` | one main file, `{% for %}` emitting N steps | single large render + single large parse |
+| `deep` | nested includes, depth N | include recursion (see caveat) |
+| `mix` | groups + jinja loop + distinct + repeated includes | realistic blend |
+
+## Reading the output
+
+```
+phase              min      median
+initial         0.1131      0.1285   <- pass 1: discover config files (no includes)
+loadtest        1.0724      1.0900   <- config fixpoint loop + full recursive include load
+build           0.1850      0.1976   <- TestSet: load_test_recursively tree build
+total           1.3886      1.4227
+counters  (last run):
+  templates :    1003 calls   0.5247s  (exclusive: jinja compile+render+tempfile)
+  yaml      :    1004 parses   1.4696s  (inclusive of nested includes)
+```
+
+- **min** is the headline (least noisy); median is a sanity check.
+- **initial / loadtest / build** map to the three pipeline stages in
+  `interpreter/process.py` and `interpreter/test_set.py`. The main file is
+  rendered+parsed across `initial` *and* `loadtest` (the loader does ~3 passes).
+- **templates** = number of `template_to_test()` calls and their *exclusive*
+  wall time (one file render each — pure jinja compile+render+tempfile I/O).
+  A high count with the same source file = recompilation, the `repeat` case.
+- **yaml** = number of `yaml_load()` parses. Its time is *inclusive* of nested
+  includes, so use the **count** for attribution, not the seconds.
+
+## Mapping to the optimisation axes
+
+| Axis (see DESIGN / discussion) | Watch | Best profile to prove it |
+|--------------------------------|-------|--------------------------|
+| 1 — cache compiled jinja templates | `templates` time drops, count unchanged | `repeat` |
+| 2 — drop the tempfile round-trip | `templates` time drops | `includes`, `repeat`, `mix` |
+| 3 — C YAML loader (libyaml) | `yaml` time / `loadtest` drops | `flat`, `jinja` |
+| 6 — O(n²) sequence splice | `build` drops | `includes`, `mix` |
+
+## How to compare before/after a change
+
+1. Run the matrix on the current code, keep the output.
+2. Apply one axis.
+3. Re-run the **same** profiles/sizes; compare `min` per phase and the counters.
+
+Change one axis at a time so the attribution is clean. Run on an idle machine
+(and note the disk: on a USB stick the tempfile round-trip of axis 2 weighs
+more).
+
+## Caveat: deep includes
+
+The loader is recursive and spends ~10 stack frames per include level, so
+`deep` hits Python's `RecursionError` around ~90 nested levels. The harness
+reports this cleanly instead of crashing. Real tests are *wide* (many steps /
+many includes), not deep, so `includes`/`repeat`/`jinja`/`mix` are the
+representative "very long" cases.
+
+## Notes
+
+- No execution is triggered — timing stops where `Batch` would mark the test
+  *loaded*.
+- The profiles contain no `<| |>`, so the external eval process is not started.
+  Pass `--with-eval` to `load_bench.py` for trees that evaluate at load time.
+- Numbers are machine- and disk-specific; only compare runs from the same host.

test/benchmark/gen_bench_test.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+"""Generate synthetic ``.tum`` test trees to benchmark *load* time.
+
+The generated trees are deliberately cheap to *build* (only ``let`` leaves and
+``group`` containers — no subprocess, no runtime side effect) so the load
+benchmark measures the parse / template / tree-build pipeline and nothing else.
+
+Profiles, each targeting a specific cost in the loader:
+
+  flat      one main file, N inline ``let`` steps, no include, no jinja.
+            Baseline: YAML parse of a big document + linear object build.
+
+  includes  main ``!include``s N *distinct* sub-files (a few steps each).
+            Stresses the per-include template+YAML+tempfile round-trip and the
+            ``sequence`` splice in test_set.load_test_recursively.
+
+  repeat    main ``!include``s the *same* parametrised leaf file N times.
+            Stresses jinja *recompilation*: the compiled template is identical
+            every time, only the render params (idx) differ -> the case a
+            template cache collapses.
+
+  jinja     one main file whose ``{% for %}`` loop emits N steps.
+            Stresses a single large jinja render + a single large YAML parse.
+
+  deep      nested includes, depth N (main -> d0 -> d1 -> ...).
+            Stresses include recursion and per-level template+YAML.
+
+  mix       a realistic blend: groups, a jinja loop, distinct includes and a
+            repeated parametrised include.
+
+Usage:
+    gen_bench_test.py --profile repeat --size 1000 --out cases/repeat_1000
+    -> writes <out>/main.tum (+ includes, + param.yaml) and prints the path.
+"""
+import argparse
+import os
+import shutil
+
+
+def _let(indent, i, name=None):
+    name = name if name is not None else f"s{i}"
+    pad = " " * indent
+    return (
+        f"{pad}- let:\n"
+        f"{pad}    name: {name}\n"
+        f"{pad}    values:\n"
+        f"{pad}        - k{i}: {i}\n"
+    )
+
+
+def gen_flat(out, n):
+    body = "".join(_let(8, i) for i in range(n))
+    main = f"main:\n    name: bench flat {n}\n    steps:\n{body}"
+    _write(out, "main.tum", main)
+
+
+def gen_includes(out, n):
+    steps = "".join(f"        - !include inc_{i}.tum\n" for i in range(n))
+    main = f"main:\n    name: bench includes {n}\n    steps:\n{steps}"
+    _write(out, "main.tum", main)
+    for i in range(n):
+        # each include is a YAML *sequence* (list of steps)
+        seq = "".join(_let(0, i * 3 + j, name=f"inc{i}_{j}") for j in range(3))
+        _write(out, f"inc_{i}.tum", seq)
+
+
+def gen_repeat(out, n):
+    steps = "".join(
+        f"        - !include {{file: leaf.tum, idx: {i}}}\n" for i in range(n)
+    )
+    main = f"main:\n    name: bench repeat {n}\n    steps:\n{steps}"
+    _write(out, "main.tum", main)
+    leaf = (
+        "- let:\n"
+        "    name: leaf_{{ idx }}\n"
+        "    values:\n"
+        "        - leaf_{{ idx }}: {{ idx }}\n"
+    )
+    _write(out, "leaf.tum", leaf)
+
+
+def gen_jinja(out, n):
+    main = (
+        f"main:\n    name: bench jinja {n}\n    steps:\n"
+        "{% for i in range(" + str(n) + ") %}\n"
+        "        - let:\n"
+        "            name: j{{ i }}\n"
+        "            values:\n"
+        "                - k{{ i }}: {{ i }}\n"
+        "{% endfor %}\n"
+    )
+    _write(out, "main.tum", main)
+
+
+def gen_deep(out, n):
+    main = (
+        f"main:\n    name: bench deep {n}\n    steps:\n"
+        "        - let:\n            name: top\n            values:\n                - a: 0\n"
+        "        - !include d_0.tum\n"
+    )
+    _write(out, "main.tum", main)
+    for i in range(n):
+        seq = _let(0, i, name=f"d{i}")
+        if i < n - 1:
+            seq += f"- !include d_{i + 1}.tum\n"
+        _write(out, f"d_{i}.tum", seq)
+
+
+def gen_mix(out, n):
+    # n groups, each: 2 inline lets, one distinct include, one repeated include,
+    # plus a small jinja loop. Roughly ~6*n steps.
+    per = max(1, n)
+    parts = [f"main:\n    name: bench mix {n}\n    steps:\n"]
+    for g in range(per):
+        parts.append(
+            f"        - group:\n"
+            f"            name: grp{g}\n"
+            f"            steps:\n"
+        )
+        parts.append(_let(16, g * 2, name=f"g{g}_a"))
+        parts.append(_let(16, g * 2 + 1, name=f"g{g}_b"))
+        parts.append(f"                - !include inc_{g}.tum\n")
+        parts.append(f"                - !include {{file: leaf.tum, idx: {g}}}\n")
+        parts.append(
+            "{% for i in range(3) %}\n"
+            f"                - let:\n"
+            f"                    name: g{g}_j{{{{ i }}}}\n"
+            f"                    values:\n"
+            f"                        - g{g}_k{{{{ i }}}}: {{{{ i }}}}\n"
+            "{% endfor %}\n"
+        )
+    _write(out, "main.tum", "".join(parts))
+    for g in range(per):
+        _write(out, f"inc_{g}.tum", _let(0, g, name=f"mixinc{g}"))
+    _write(
+        out,
+        "leaf.tum",
+        "- let:\n    name: mixleaf_{{ idx }}\n    values:\n        - mixleaf_{{ idx }}: {{ idx }}\n",
+    )
+
+
+PROFILES = {
+    "flat": gen_flat,
+    "includes": gen_includes,
+    "repeat": gen_repeat,
+    "jinja": gen_jinja,
+    "deep": gen_deep,
+    "mix": gen_mix,
+}
+
+
+def _write(out, name, content):
+    with open(os.path.join(out, name), "w") as f:
+        f.write(content)
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--profile", required=True, choices=sorted(PROFILES))
+    ap.add_argument("--size", type=int, default=1000,
+                    help="profile-specific count (steps / includes / depth)")
+    ap.add_argument("--out", required=True, help="output directory (recreated)")
+    args = ap.parse_args()
+
+    out = os.path.abspath(args.out)
+    if os.path.isdir(out):
+        shutil.rmtree(out)
+    os.makedirs(out)
+
+    # minimal config file so the loader does not emit "no param file" noise
+    _write(out, "param.yaml", "bench_dummy: 1\n")
+
+    PROFILES[args.profile](out, args.size)
+    print(os.path.join(out, "main.tum"))
+
+
+if __name__ == "__main__":
+    main()

test/benchmark/load_bench.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+"""Time the testium *load* pipeline on a given ``.tum`` tree.
+
+It drives the real loader code (``TestProcess._load_initial_params`` /
+``_load_test`` then ``TestSet(...)``) in-process, so the numbers track the
+production path and stay honest as the code evolves. Execution is never
+triggered — we stop exactly where ``Batch`` would report the test as *loaded*.
+
+Reported per run, over ``--repeat`` iterations (min is the headline, least
+noisy):
+
+    initial   first pass: discover config files (template+YAML, no includes)
+    loadtest  config-file fixpoint loop + full recursive include/template/YAML
+    build     TestSet construction: the load_test_recursively tree build
+    total     sum of the three
+
+Plus instrumentation counters (exact call counts, wall time) for the two
+hot leaves the optimisation axes target:
+
+    templates  jinja template_to_test() calls   (axis 1 compile cache, axis 2 tempfile)
+    yaml       yaml_load() parses               (axis 3 C loader)
+
+template time is exclusive (one file render); yaml time is wall-inclusive of
+nested includes, so lean on the *counts* for attribution.
+
+Must run inside the project venv (jinja2, pyyaml, telnetlib3, ...). The
+benchmark profiles contain no ``<| |>`` so the external eval process is not
+needed; pass --with-eval to start it for faithfulness on eval-heavy trees.
+
+Usage (see run.sh for the convenience wrapper):
+    test/tmp/.venv/bin/python3 test/benchmark/load_bench.py [--repeat 5] <main.tum>
+"""
+import argparse
+import os
+import statistics
+import sys
+from queue import Queue
+from time import perf_counter
+
+# --- bootstrap: src/testium for flat imports, src for `import testium` --------
+HERE = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
+sys.path.insert(0, os.path.join(ROOT, "src"))
+sys.path.insert(0, os.path.join(ROOT, "src", "testium"))
+
+import api.testium as tm  # noqa: E402
+from interpreter.utils.test_init import env_init, apply_overrides  # noqa: E402
+from interpreter.utils.test_ctrl import TestSetController  # noqa: E402
+from interpreter.process import TestProcess  # noqa: E402
+from interpreter.test_set import TestSet  # noqa: E402
+from interpreter.utils.py_eval import eval_process_init  # noqa: E402
+from interpreter.utils.api_srv import api_request  # noqa: E402
+
+# --- instrumentation: count + time the two hot leaves -------------------------
+import interpreter.process as _proc  # noqa: E402
+import interpreter.utils.include as _inc  # noqa: E402
+import interpreter.utils.test_init as _ti  # noqa: E402
+import interpreter.utils.template as _tpl  # noqa: E402
+import interpreter.utils.yaml_load as _yl  # noqa: E402
+
+_C = {"tpl_n": 0, "tpl_t": 0.0, "yaml_n": 0, "yaml_t": 0.0}
+_orig_tpl = _tpl.template_to_test
+_orig_yaml = _yl.yaml_load
+
+
+def _wrap_tpl(*a, **k):
+    t = perf_counter()
+    try:
+        return _orig_tpl(*a, **k)
+    finally:
+        _C["tpl_t"] += perf_counter() - t
+        _C["tpl_n"] += 1
+
+
+def _wrap_yaml(*a, **k):
+    t = perf_counter()
+    try:
+        return _orig_yaml(*a, **k)
+    finally:
+        _C["yaml_t"] += perf_counter() - t
+        _C["yaml_n"] += 1
+
+
+# rebind in every module that did `from ... import template_to_test / yaml_load`
+for _m in (_proc, _inc):
+    _m.template_to_test = _wrap_tpl
+for _m in (_proc, _inc, _ti):
+    _m.yaml_load = _wrap_yaml
+
+
+def _reset_counters():
+    _C.update(tpl_n=0, tpl_t=0.0, yaml_n=0, yaml_t=0.0)
+
+
+def load_once(tp, fname, test_dir):
+    """One full load (no execution). Returns (initial, loadtest, build) seconds."""
+    t0 = perf_counter()
+    init_pf, gv = tp._load_initial_params(test_dir)
+    t1 = perf_counter()
+    test_dict, _pf = tp._load_test(init_pf, gv)
+    t2 = perf_counter()
+    TestSet(fname, test_dict, Queue())
+    t3 = perf_counter()
+    return (t1 - t0, t2 - t1, t3 - t2)
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("main_tum", help="path to the generated main.tum")
+    ap.add_argument("--repeat", type=int, default=5)
+    ap.add_argument("--with-eval", action="store_true",
+                    help="start the external eval process (needed only for <| |> at load)")
+    ap.add_argument("--quiet", action="store_true",
+                    help="silence the loader's INFO output during runs")
+    args = ap.parse_args()
+
+    fname = os.path.abspath(args.main_tum)
+    if not os.path.isfile(fname):
+        ap.error(f"not found: {fname}")
+    test_dir = os.path.dirname(fname)
+
+    env_init()
+    apply_overrides({}, {})
+
+    eval_proc = None
+    if args.with_eval:
+        eval_proc = eval_process_init(api_request, 10, test_dir)
+        eval_proc.start()
+        eval_proc.wait_ready(10)
+
+    if args.quiet:
+        # the loader prints a couple of INFO lines per config file; mute stdout
+        # around the measured section to avoid I/O skew.
+        devnull = open(os.devnull, "w")
+        real_stdout = sys.stdout
+
+    tp = TestProcess(fname, Queue(), TestSetController(),
+                     config_files=[], defines={}, gui_defaults={}, text_mode=True)
+
+    samples = []  # list of (initial, loadtest, build)
+    last_counters = None
+    try:
+        for r in range(args.repeat):
+            _reset_counters()
+            if args.quiet:
+                sys.stdout = devnull
+            try:
+                samples.append(load_once(tp, fname, test_dir))
+            except RecursionError:
+                if args.quiet:
+                    sys.stdout = real_stdout
+                print(f"file      : {fname}")
+                print("ERROR     : RecursionError during load — the include "
+                      "nesting is too deep for the recursive loader.\n"
+                      "            (each include level costs ~10 stack frames; "
+                      "raise sys.setrecursionlimit to probe further.)")
+                return 2
+            except Exception as e:  # noqa: BLE001 - report, don't crash the bench
+                if args.quiet:
+                    sys.stdout = real_stdout
+                print(f"file      : {fname}")
+                print(f"ERROR     : load failed: {type(e).__name__}: {e}")
+                return 2
+            finally:
+                if args.quiet:
+                    sys.stdout = real_stdout
+            last_counters = dict(_C)
+    finally:
+        if eval_proc is not None:
+            eval_proc.stop()
+            eval_proc.join()
+        if args.quiet:
+            devnull.close()
+
+    initial = [s[0] for s in samples]
+    loadtest = [s[1] for s in samples]
+    build = [s[2] for s in samples]
+    total = [sum(s) for s in samples]
+
+    def stat(xs):
+        return min(xs), statistics.median(xs)
+
+    print(f"file      : {fname}")
+    print(f"repeats   : {args.repeat}   (showing  min | median, seconds)")
+    print(f"{'phase':<10}{'min':>12}{'median':>12}")
+    for name, xs in (("initial", initial), ("loadtest", loadtest),
+                     ("build", build), ("total", total)):
+        mn, md = stat(xs)
+        print(f"{name:<10}{mn:>12.4f}{md:>12.4f}")
+    if last_counters:
+        print("counters  (last run):")
+        print(f"  templates : {last_counters['tpl_n']:>7d} calls   "
+              f"{last_counters['tpl_t']:>8.4f}s  (exclusive: jinja compile+render+tempfile)")
+        print(f"  yaml      : {last_counters['yaml_n']:>7d} parses  "
+              f"{last_counters['yaml_t']:>8.4f}s  (inclusive of nested includes)")
+
+
+if __name__ == "__main__":
+    sys.exit(main() or 0)

test/benchmark/run.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Load-time benchmark driver: generate synthetic .tum trees and time the
+# testium load pipeline on them, using the project venv.
+#
+# Usage:
+#   ./test/benchmark/run.sh                     # default matrix (all profiles)
+#   ./test/benchmark/run.sh <profile> <size>    # one profile at one size
+#   REPEAT=10 ./test/benchmark/run.sh repeat 2000
+#
+# Profiles: flat includes repeat jinja deep mix   (see gen_bench_test.py)
+#
+# Generated trees go under test/benchmark/cases/ (git-ignored). The numbers
+# are wall-clock; run on an otherwise idle machine and compare min values.
+set -e
+
+SCRIPT_DIR="$(realpath "$(dirname "$(readlink -f "$0")")")"
+PROJECT_DIR="$(realpath "$SCRIPT_DIR/../..")"
+VPY="$PROJECT_DIR/test/tmp/.venv/bin/python3"
+CASES="$SCRIPT_DIR/cases"
+REPEAT="${REPEAT:-5}"
+
+if [ ! -x "$VPY" ]; then
+    echo "ERROR: project venv not found at $VPY — run ./run.sh once to create it." >&2
+    exit 1
+fi
+
+bench() {
+    local profile="$1" size="$2"
+    local out="$CASES/${profile}_${size}"
+    local main
+    main="$("$VPY" "$SCRIPT_DIR/gen_bench_test.py" --profile "$profile" --size "$size" --out "$out")"
+    echo "===== profile=$profile size=$size ====="
+    "$VPY" "$SCRIPT_DIR/load_bench.py" --repeat "$REPEAT" --quiet "$main"
+    echo
+}
+
+if [ $# -eq 2 ]; then
+    bench "$1" "$2"
+    exit 0
+fi
+
+# Default matrix. 'deep' is kept small: the recursive loader hits Python's
+# recursion limit around ~90 nested include levels.
+bench flat     2000
+bench includes 1000
+bench repeat   1000
+bench jinja    2000
+bench deep     40
+bench mix      300

test/robustness/console serial/rsl_terminal_robustness.tum

@@ -1,69 +0,0 @@
-# Main
-################################################################################
-main:
-  name:                               Serial Terminal bug reproducer
-  version:                            0.1
-  steps:
-    - group:
-        name:                         Test preparation
-        steps:
-          - console:
-              name:             Open RSL Simulator Terminal
-              console_name:     RSL_simulator
-              steps:
-                  - open:
-                      protocol:   terminal
-                      terminal_path: $(rslsimulatorpath)
-                  - writeln: "pwd"
-                  - read_until: {expected: "$", timeout: 5}
-                  - writeln: "./RSverify $(rsTx)" # /dev/ttyMUE1
-                  - read_until: {expected: "RSL controller>", timeout: 5}
-                  - writeln: "setportconf 0 115200 none 8 1 1 255"
-                  - read_until: {expected: "RSL controller>", timeout: 5}
-                  - writeln: "send4ever 0 0"
-                  - read_until: {expected: "RSL controller>", timeout: 5}
-
-          - console:
-              name:                   Open the EUT console
-              console_name:           cons_target
-              doc:                    Initiates the console of the target in order
-                                      to be ready to capture its traces.
-              stop_on_failure:        True
-              steps:
-                - open:
-                    protocol:         serial
-                    serial_port:      $(rsRx) # /dev/ttyMUE2
-                    serial_baudrate:  115200
-
-    - loop:
-        name:                         Qualification loop
-        stop_on_failure:              False
-        steps:
-          - py_func:
-              name:                   Capture the RS serial output
-              file:                   $(test_directory)/terminal_bug_reproducer.py
-              func_name:              RetreiveData
-              param:
-                - cons_target
-
-          - sleep: {timeout: 1}
-
-# Cleanup sequence
-#-------------------------------------------------------------------------------
-    - group:
-        name:                         Cleanup
-        execute_on_stop:              True
-        steps:
-          - console:
-              name:                   Close the target console
-              console_name:           cons_target
-              execute_on_stop:        True
-              steps:
-                - close:
-
-          - console:
-              name:                   Close the RSL_simulator
-              console_name:           RSL_simulator
-              execute_on_stop:        True
-              steps:
-                - close:

test/robustness/console serial/terminal_bug_reproducer.py

@@ -1,26 +0,0 @@
-import api.testium as tm
-
-def RetreiveData(console_name):
-    print("--------------- retrieving data ---------------")
-    result = 0
-    cons   = tm.console(console_name)
-
-    if cons is None:
-        print("--------------- The console does not exist ---------------")
-    else:
-        try:
-            is_finished = False
-            while not is_finished:
-                status, d = cons.read_until('\n', timeout=0, return_data=True, mute=True)
-                if 0 == status:
-                    print("--------------- Data ---------------")
-                    print(d)
-                else:
-                    print("--------------- No data ---------------")
-                    print("Status: ", status)
-                    is_finished = True
-        except:
-            print("--------------- Error retrieving data ---------------")
-            result = -1
-
-    return result

test/robustness/console terminal/generate_char.sh

@@ -1,9 +0,0 @@
-chars='<=>| -,;:!/."()[]{}*\&#%+012345689abcdefghiklmnopqrstuvwxyzABCD'
-for j in {1..256} ;
-do
-    for i in {1..256} ; do
-        echo -n "${chars:RANDOM%${#chars}:1}"
-    done
-    echo
-    sleep 0.01
-done

test/robustness/console terminal/terminal_bug_reproducer.py

@@ -1,26 +0,0 @@
-import api.testium as tm
-
-def RetreiveData(console_name):
-    print("--------------- retrieving data ---------------")
-    result = 0
-    cons   = tm.console(console_name)
-
-    if cons is None:
-        print("--------------- The console does not exist ---------------")
-    else:
-        try:
-            is_finished = False
-            while not is_finished:
-                status, d = cons.read_until('\n', timeout=0, return_data=True, mute=True)
-                if 0 == status:
-                    print("--------------- Data ---------------")
-                    print(d)
-                else:
-                    print("--------------- No data ---------------")
-                    print("Status: ", status)
-                    is_finished = True
-        except:
-            print("--------------- Error retrieving data ---------------")
-            result = -1
-
-    return result

test/robustness/console terminal/terminal_robustness.tum

@@ -1,50 +0,0 @@
-# Main
-################################################################################
-main:
-  name:                               Terminal bug reproducer
-  version:                            0.1
-  steps:
-    - group:
-        name:                         Test preparation
-        steps:
-          - console:
-              name:                   Open the EUT console
-              console_name:           cons_target
-              doc:                    Initiates the console of the target in order
-                                      to be ready to capture its traces.
-              stop_on_failure:        True
-              steps:
-                - open:
-                    protocol: terminal
-
-    - loop:
-        name:                         Qualification loop
-        stop_on_failure:              False
-        steps:
-          - console:
-              name:                   write random data
-              console_name:           cons_target
-              steps:
-                  - writeln: bash $(test_directory)/generate_char.sh
-
-          - py_func:
-              name:                   Capture the terminal output
-              file:                   $(test_directory)/terminal_bug_reproducer.py
-              func_name:              RetreiveData
-              param:
-                - cons_target
-
-          - sleep: {timeout: 1}
-
-# Cleanup sequence
-#-------------------------------------------------------------------------------
-    - group:
-        name:                         Cleanup
-        execute_on_stop:              True
-        steps:
-          - console:
-              name:                   Close the target console
-              console_name:           cons_target
-              execute_on_stop:        True
-              steps:
-                - close: