From 5086aa6c0ebefad91acb5298e5e66887a8df13ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Sun, 31 May 2026 10:41:42 +0200 Subject: [PATCH] perf(load): cache compiled jinja templates, render in memory Shared jinja Environment + compiled-template cache keyed on (path, mtime, size), and render to an in-memory StringIO instead of a temp file. Behaviour unchanged (validation suite passes); template time -10..40x, total load -20..30% on template-heavy trees. Co-Authored-By: Claude Opus 4.8 --- src/testium/interpreter/utils/template.py | 84 ++++++++++++++++------ src/testium/interpreter/utils/yaml_load.py | 4 +- 2 files changed, 64 insertions(+), 24 deletions(-) diff --git a/src/testium/interpreter/utils/template.py b/src/testium/interpreter/utils/template.py index c7a3173..a6d3d91 100644 --- a/src/testium/interpreter/utils/template.py +++ b/src/testium/interpreter/utils/template.py @@ -1,33 +1,74 @@ +import io import os from sys import exc_info -from jinja2 import Template +from jinja2 import Environment from jinja2.exceptions import TemplateSyntaxError, TemplateError, UndefinedError -from tempfile import TemporaryFile from interpreter.utils.yaml_load import print_yaml from runtime.tum_except import ETUMSyntaxError +# One Environment reused for every render (default settings, i.e. identical +# behaviour to jinja2.Template), plus a compiled-template cache so a file that +# is included many times — or a test that is reloaded — is compiled only once. +# Jinja compilation is the expensive step; render (variable substitution) stays +# per-call. Cache is keyed on path + mtime + size so an edited file recompiles. +_ENV = Environment() +_template_cache = {} # abspath -> (mtime_ns, size, compiled_template) + + +class _RenderedStream(io.StringIO): + """A rendered template kept in memory. + + Carries ``root`` (and ``name``) so the YAML loader resolves ``!include`` + paths exactly as it did from the on-disk temp file this replaces — without + the write + seek + read round-trip (one temp file per included file). That + round-trip is pure overhead, and especially costly on slow storage. + """ + + +def _compiled_template(filename: str): + """Return the compiled jinja template for *filename*, reusing the cached + one when the file is unchanged (path + mtime + size).""" + key = os.path.abspath(filename) + try: + st = os.stat(filename) + except OSError: + st = None + if st is not None: + cached = _template_cache.get(key) + if (cached is not None + and cached[0] == st.st_mtime_ns + and cached[1] == st.st_size): + return cached[2] + with open(filename, "r") as f: + source = f.read() + template = _ENV.from_string(source) # compile (may raise TemplateSyntaxError) + if st is not None: + _template_cache[key] = (st.st_mtime_ns, st.st_size, template) + return template + + def template_to_test(filename: str, params: list): """ Function which processes an eventual jinja2 template to a test file """ - # Temporary file created to receive the processed include - # file - tmpf = TemporaryFile('w+t') - with open(filename, 'r') as f: - try: - j2_template = Template(f.read()) - except TemplateError as e: + # Compile (cached) — a syntax error in the template surfaces here. + try: + j2_template = _compiled_template(filename) + except TemplateError as e: + with open(filename, "r") as f: print_yaml(f, filename) - type, value, tb = exc_info() - msg = "Template error" - if hasattr(value, 'lineno'): - msg = msg + f" on line {value.lineno}: " - else: - msg += ": " - raise ETUMSyntaxError(msg + str(e), filename) + type, value, tb = exc_info() + msg = "Template error" + if hasattr(value, 'lineno'): + msg = msg + f" on line {value.lineno}: " + else: + msg += ": " + raise ETUMSyntaxError(msg + str(e), filename) + + # Render into memory (no temp file). try: params["include_directory"] = os.path.dirname(os.path.abspath(filename)) - tmpf.write(j2_template.render(params)) + rendered = j2_template.render(params) except TemplateSyntaxError as e: raise ETUMSyntaxError(f"""Template loading of file '{filename}' with following parameters '{str(params)}' Syntax error in template: {e.message}""") @@ -42,8 +83,7 @@ Template rendering error: {e.message}""") raise ETUMSyntaxError(f"""Template loading of file '{filename}' with following parameters '{str(params)}' Unexpected error: {str(e)}""") - # return to begining of the temp file - tmpf.seek(0, os.SEEK_SET) - tmpf.root = os.path.dirname(filename) - - return tmpf \ No newline at end of file + stream = _RenderedStream(rendered) + stream.root = os.path.dirname(filename) + stream.name = filename + return stream diff --git a/src/testium/interpreter/utils/yaml_load.py b/src/testium/interpreter/utils/yaml_load.py index 0502c19..df6b82f 100644 --- a/src/testium/interpreter/utils/yaml_load.py +++ b/src/testium/interpreter/utils/yaml_load.py @@ -21,10 +21,10 @@ def yaml_load(file, real_file_name: str, loader: Loader): return load(file, loader) except ParserError as e: - if isinstance(file, io.TextIOWrapper): + if isinstance(file, (io.TextIOWrapper, io.StringIO)): print_yaml(file, real_file_name) raise ETUMSyntaxError(f"yaml file parsing error: " + str(e), real_file_name) except ScannerError as e: - if isinstance(file, io.TextIOWrapper): + if isinstance(file, (io.TextIOWrapper, io.StringIO)): print_yaml(file, real_file_name) raise ETUMSyntaxError("yaml file scanning error: " + str(e), real_file_name)