v-and-v/testium
2
0
Fork 0
Code Issues 5 Pull Requests 1 Actions Packages Projects Releases 7 Wiki Activity

test: add load-time benchmark (jinja/include trees)

Browse Source 
Generator + in-process harness timing the real loader's three stages and
template/YAML call counts, across tunable profiles. cases/ git-ignored;
see test/benchmark/README.md.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Foué
2026-05-31 10:41:42 +02:00
parent 6e31ae971a
commit ef49789780
5 changed files with 545 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

179
test/benchmark/gen_bench_test.py Executable file
View File

@@ -0,0 +1,179 @@
#!/usr/bin/env python3
"""Generate synthetic ``.tum`` test trees to benchmark *load* time.
The generated trees are deliberately cheap to *build* (only ``let`` leaves and
``group`` containers — no subprocess, no runtime side effect) so the load
benchmark measures the parse / template / tree-build pipeline and nothing else.
Profiles, each targeting a specific cost in the loader:
flat one main file, N inline ``let`` steps, no include, no jinja.
Baseline: YAML parse of a big document + linear object build.
includes main ``!include``s N *distinct* sub-files (a few steps each).
Stresses the per-include template+YAML+tempfile round-trip and the
``sequence`` splice in test_set.load_test_recursively.
repeat main ``!include``s the *same* parametrised leaf file N times.
Stresses jinja *recompilation*: the compiled template is identical
every time, only the render params (idx) differ -> the case a
template cache collapses.
jinja one main file whose ``{% for %}`` loop emits N steps.
Stresses a single large jinja render + a single large YAML parse.
deep nested includes, depth N (main -> d0 -> d1 -> ...).
Stresses include recursion and per-level template+YAML.
mix a realistic blend: groups, a jinja loop, distinct includes and a
repeated parametrised include.
Usage:
gen_bench_test.py --profile repeat --size 1000 --out cases/repeat_1000
-> writes <out>/main.tum (+ includes, + param.yaml) and prints the path.
"""
import argparse
import os
import shutil
def _let(indent, i, name=None):
name = name if name is not None else f"s{i}"
pad = " " * indent
return (
f"{pad}- let:\n"
f"{pad} name: {name}\n"
f"{pad} values:\n"
f"{pad} - k{i}: {i}\n"
)
def gen_flat(out, n):
body = "".join(_let(8, i) for i in range(n))
main = f"main:\n name: bench flat {n}\n steps:\n{body}"
_write(out, "main.tum", main)
def gen_includes(out, n):
steps = "".join(f" - !include inc_{i}.tum\n" for i in range(n))
main = f"main:\n name: bench includes {n}\n steps:\n{steps}"
_write(out, "main.tum", main)
for i in range(n):
# each include is a YAML *sequence* (list of steps)
seq = "".join(_let(0, i * 3 + j, name=f"inc{i}_{j}") for j in range(3))
_write(out, f"inc_{i}.tum", seq)
def gen_repeat(out, n):
steps = "".join(
f" - !include {{file: leaf.tum, idx: {i}}}\n" for i in range(n)
)
main = f"main:\n name: bench repeat {n}\n steps:\n{steps}"
_write(out, "main.tum", main)
leaf = (
"- let:\n"
" name: leaf_{{ idx }}\n"
" values:\n"
" - leaf_{{ idx }}: {{ idx }}\n"
)
_write(out, "leaf.tum", leaf)
def gen_jinja(out, n):
main = (
f"main:\n name: bench jinja {n}\n steps:\n"
"{% for i in range(" + str(n) + ") %}\n"
" - let:\n"
" name: j{{ i }}\n"
" values:\n"
" - k{{ i }}: {{ i }}\n"
"{% endfor %}\n"
)
_write(out, "main.tum", main)
def gen_deep(out, n):
main = (
f"main:\n name: bench deep {n}\n steps:\n"
" - let:\n name: top\n values:\n - a: 0\n"
" - !include d_0.tum\n"
)
_write(out, "main.tum", main)
for i in range(n):
seq = _let(0, i, name=f"d{i}")
if i < n - 1:
seq += f"- !include d_{i + 1}.tum\n"
_write(out, f"d_{i}.tum", seq)
def gen_mix(out, n):
# n groups, each: 2 inline lets, one distinct include, one repeated include,
# plus a small jinja loop. Roughly ~6*n steps.
per = max(1, n)
parts = [f"main:\n name: bench mix {n}\n steps:\n"]
for g in range(per):
parts.append(
f" - group:\n"
f" name: grp{g}\n"
f" steps:\n"
)
parts.append(_let(16, g * 2, name=f"g{g}_a"))
parts.append(_let(16, g * 2 + 1, name=f"g{g}_b"))
parts.append(f" - !include inc_{g}.tum\n")
parts.append(f" - !include {{file: leaf.tum, idx: {g}}}\n")
parts.append(
"{% for i in range(3) %}\n"
f" - let:\n"
f" name: g{g}_j{{{{ i }}}}\n"
f" values:\n"
f" - g{g}_k{{{{ i }}}}: {{{{ i }}}}\n"
"{% endfor %}\n"
)
_write(out, "main.tum", "".join(parts))
for g in range(per):
_write(out, f"inc_{g}.tum", _let(0, g, name=f"mixinc{g}"))
_write(
out,
"leaf.tum",
"- let:\n name: mixleaf_{{ idx }}\n values:\n - mixleaf_{{ idx }}: {{ idx }}\n",
)
PROFILES = {
"flat": gen_flat,
"includes": gen_includes,
"repeat": gen_repeat,
"jinja": gen_jinja,
"deep": gen_deep,
"mix": gen_mix,
}
def _write(out, name, content):
with open(os.path.join(out, name), "w") as f:
f.write(content)
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--profile", required=True, choices=sorted(PROFILES))
ap.add_argument("--size", type=int, default=1000,
help="profile-specific count (steps / includes / depth)")
ap.add_argument("--out", required=True, help="output directory (recreated)")
args = ap.parse_args()
out = os.path.abspath(args.out)
if os.path.isdir(out):
shutil.rmtree(out)
os.makedirs(out)
# minimal config file so the loader does not emit "no param file" noise
_write(out, "param.yaml", "bench_dummy: 1\n")
PROFILES[args.profile](out, args.size)
print(os.path.join(out, "main.tum"))
if __name__ == "__main__":
main()