validation: dedicated venv + fix python_bin override timing

eval_proc was started before -d/GUI defines reached gd, so
``-d python_bin=...`` and the GUI ``python_bin`` preference were
silently ignored by the very subprocess that runs ``<| ... |>`` evals
(and only took effect for later items once the discovery cache had
already been seeded with the system interpreter). apply_overrides() is
now applied before eval_process_init(), and bins._resolve()'s cache is
keyed by (name, override) so a later param.yaml change re-resolves on
the next lookup.

The validation suite now ships a wrapper (run.sh / run.bat) that
creates a dedicated venv in the system temp dir and pins it via
``-d python_bin=...``. A new ``venv`` item asserts the override took
effect for both eval_proc and py_func paths, with a
``sys.prefix != sys.base_prefix`` marker to catch the case where the
override happens to be a system interpreter (path-equality alone would
miss it, the venv's ``bin/python3`` being a symlink to the host).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-05-19 08:19:57 +02:00
parent 6f832cd67b
commit 4d8cafb5a0
10 changed files with 321 additions and 17 deletions

View File

@@ -0,0 +1 @@
no_param: Null

View File

@@ -0,0 +1,53 @@
# venv test: assert that the dedicated validation venv is the python
# being used for every test-execution subprocess (eval_proc / py_func /
# cycle / ...). The venv path is pinned by ``-d python_bin=...`` in
# test/validation/run.sh (or run.bat).
#
# We use ``abspath``/``normpath`` rather than ``realpath`` on purpose:
# the venv's ``bin/python3`` is a symlink to the host python, so
# realpath would map every venv interpreter to the same system path and
# the comparison would silently pass even *without* a venv.
# ``sys.prefix != sys.base_prefix`` is the venv-specific marker that
# catches that case.
- check:
name: python_bin is set in the global dict
key: $(test)_PASS
values:
- <| bool(r"$(python_bin)") |>
- check:
name: eval_proc subprocess runs in the validation venv
key: $(test)_PASS
values:
- <| os.path.normpath(os.path.abspath(sys.executable)) == os.path.normpath(os.path.abspath(r"$(python_bin)")) |>
- check:
name: eval_proc sys.prefix matches python_bin venv root
key: $(test)_PASS
values:
- <| os.path.normpath(os.path.abspath(sys.prefix)) == os.path.dirname(os.path.dirname(os.path.normpath(os.path.abspath(r"$(python_bin)")))) |>
- check:
name: eval_proc is actually inside a venv (sys.prefix != sys.base_prefix)
key: $(test)_PASS
values:
- <| os.path.normpath(os.path.abspath(sys.prefix)) != os.path.normpath(os.path.abspath(sys.base_prefix)) |>
- py_func:
name: py_func subprocess runs in the validation venv
key: $(test)_PASS
file: $(test_path)$(psep)verify_venv.py
func_name: check_sys_executable
- py_func:
name: py_func sys.prefix matches python_bin venv root
key: $(test)_PASS
file: $(test_path)$(psep)verify_venv.py
func_name: check_sys_prefix_in_venv
- py_func:
name: py_func is actually inside a venv
key: $(test)_PASS
file: $(test_path)$(psep)verify_venv.py
func_name: check_is_venv

View File

@@ -0,0 +1,62 @@
import os
import sys
import py_func.tm as tm
def _norm(p):
# normpath + normcase, without resolving symlinks. realpath() would
# follow the venv's ``python3`` symlink to ``/usr/bin/python3.X`` and
# defeat the comparison.
return os.path.normcase(os.path.normpath(os.path.abspath(p)))
def _venv_dir():
# python_bin is at ``<venv>/(bin|Scripts)/python*`` so the venv root
# is two levels above the executable.
exe = tm.gd("python_bin", "")
if not exe:
return ""
return os.path.dirname(os.path.dirname(_norm(exe)))
def check_sys_executable():
"""py_func subprocess: sys.executable must match the configured python_bin."""
expected = _norm(tm.gd("python_bin", ""))
actual = _norm(sys.executable)
if expected and actual == expected:
return True
return (
-1,
f"sys.executable={actual!r} differs from python_bin={expected!r}",
)
def check_sys_prefix_in_venv():
"""py_func subprocess: sys.prefix must match the venv root derived
from python_bin (two levels up from the executable)."""
venv = _venv_dir()
if not venv:
return (-1, "python_bin is not set in the global dict")
actual = _norm(sys.prefix)
if actual == venv:
return True
return (
-1,
f"sys.prefix={actual!r} is not the validation venv {venv!r}",
)
def check_is_venv():
"""py_func subprocess: confirm we are inside a venv, i.e. sys.prefix
differs from sys.base_prefix. This catches the case where python_bin
happens to be a system interpreter and the path-based check would
pass trivially."""
actual = _norm(sys.prefix)
base = _norm(sys.base_prefix)
if actual != base:
return True
return (
-1,
f"sys.prefix == sys.base_prefix == {actual!r}: not running in a venv",
)