From 7954f2cb2b431adf7efc1466e6faa9bd36192129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Dausseur?= Date: Sat, 3 Jan 2026 19:29:14 +0100 Subject: [PATCH] doc and various fixes of lua for windows --- doc/examples/param.yaml | 2 +- doc/manual/sphinx/source/overview.rst | 2 +- .../source/test_items/lua_func_test_item.rst | 23 +++- .../source/test_items/py_func_test_item.rst | 10 +- doc/manual/sphinx/source/tum_syntax.rst | 83 +++++++----- src/testium/interpreter/utils/jrpc.py | 93 +++++++++++-- .../interpreter/utils/lua_func_exec.py | 127 ++++++++++++++++-- src/testium/interpreter/utils/paths.py | 1 - src/testium/interpreter/utils/py_func_exec.py | 5 +- 9 files changed, 283 insertions(+), 63 deletions(-) diff --git a/doc/examples/param.yaml b/doc/examples/param.yaml index d8d2dc4..73a2c04 100644 --- a/doc/examples/param.yaml +++ b/doc/examples/param.yaml @@ -17,7 +17,7 @@ python_path_Linux: $(home)/tmp/tum_venv/bin/python3 LUA_PATH_Linux: /usr/share/lua/5.4/?.lua;/usr/local/share/lua/5.4/?.lua;/usr/local/share/lua/5.4/?/init.lua;/usr/share/lua/5.4/?/init.lua;/usr/local/lib/lua/5.4/?.lua;/usr/local/lib/lua/5.4/?/init.lua;/usr/lib/lua/5.4/?.lua;/usr/lib/lua/5.4/?/init.lua;./?.lua;./?/init.lua;/home/francois/.luarocks/share/lua/5.4/?.lua;/home/francois/.luarocks/share/lua/5.4/?/init.lua LUA_CPATH_Linux: /usr/local/lib/lua/5.4/?.so;/usr/lib/lua/5.4/?.so;/usr/local/lib/lua/5.4/loadall.so;/usr/lib/lua/5.4/loadall.so;./?.so;/home/francois/.luarocks/lib/lua/5.4/?.so -PATH_Linux: "" +PATH_Linux: LUA_PATH_Windows: ;.\?.lua;C:\Lua\5.1\lua\?.lua;C:\Lua\5.1\lua\?\init.lua;C:\Lua\5.1\?.lua;C:\Lua\5.1\?\init.lua;C:\Lua\5.1\lua\?.luac LUA_CPATH_Windows: .\?.dll;C:\Lua\5.1\?.dll;C:\Lua\5.1\loadall.dll;C:\Lua\5.1\clibs\?.dll;C:\Lua\5.1\clibs\loadall.dll;.\?51.dll;C:\Lua\5.1\?51.dll;C:\Lua\5.1\clibs\?51.dll diff --git a/doc/manual/sphinx/source/overview.rst b/doc/manual/sphinx/source/overview.rst index 465d2fe..3eeb9de 100644 --- a/doc/manual/sphinx/source/overview.rst +++ b/doc/manual/sphinx/source/overview.rst @@ -13,7 +13,7 @@ Tests reports generation and customization are also in this tool's scope. Its main features are: * YAML test description, -* Test configuration files in YAML, JSON or XML, +* Test configuration files in YAML, * Full range of pre-existing Test items, * Test steps, loops, * Dynamic variables expansion at test runtime, diff --git a/doc/manual/sphinx/source/test_items/lua_func_test_item.rst b/doc/manual/sphinx/source/test_items/lua_func_test_item.rst index 2ccd022..7b95d85 100644 --- a/doc/manual/sphinx/source/test_items/lua_func_test_item.rst +++ b/doc/manual/sphinx/source/test_items/lua_func_test_item.rst @@ -62,4 +62,25 @@ on how to access to global variables from test items and scripts). In the example above, the global variable ``$(lfn_activity)`` would be created at the end of the item execution. It would contain the resulting -value of the funcToBeExecuted python function. \ No newline at end of file +value of the funcToBeExecuted python function. + +**Global variables** + +Some global variables have an impact on the ``lua_func`` test item behavior: + +* ``lua_path``: This optional global variable can be used to define + the lua executable path. If not defined, the lua interpreter is + searched in at the default place in the system. +* ``lua_env``: This global variable can be used to define + environment variables for the lua script execution environment. + Only `PATH`, `LUA_PATH`, and `LUA_CPATH` are supported. + + .. code-block:: yaml + :caption: example of configuration file: param.yaml + + [...] + lua_env: + PATH: "/my/path/" + LUA_PATH: "/my/lua/modules/?.lua;;" + LUA_CPATH: "/my/lua/modules/?.so;;" + [...] diff --git a/doc/manual/sphinx/source/test_items/py_func_test_item.rst b/doc/manual/sphinx/source/test_items/py_func_test_item.rst index cf6d178..55ec449 100644 --- a/doc/manual/sphinx/source/test_items/py_func_test_item.rst +++ b/doc/manual/sphinx/source/test_items/py_func_test_item.rst @@ -105,4 +105,12 @@ on how to access to global variables from test items and scripts). In the example above, the global variable ``$(pfn_function test item)`` would be created at the end of the item execution. It would contain the resulting -value of the funcToBeExecuted python function. \ No newline at end of file +value of the funcToBeExecuted python function. + +**Global variables** + +Some global variables have an impact on the ``py_func`` test item behavior: + +* ``python_path``: This optional global variable can be used to define + the python executable path. If not defined, the python interpreter is + searched in at the default places in the system. diff --git a/doc/manual/sphinx/source/tum_syntax.rst b/doc/manual/sphinx/source/tum_syntax.rst index 335f984..f5e8c6b 100644 --- a/doc/manual/sphinx/source/tum_syntax.rst +++ b/doc/manual/sphinx/source/tum_syntax.rst @@ -36,60 +36,52 @@ The example below shows a basic implementation of the TUM description file: Configuration files -------------------- -A configuration file can be specified in the .tum file or by the command line. -This configuration file is optional. +A configuration file can be specified in the `.tum` file or by the command line. +This configuration file is optional and must be a YAML file. -It can be of three different syntax: - -* XML -* YAML -* JSON - -The type of file is recognized by the file name extension (.xml, .yaml, .json). +The type of file is recognized by the file name extension `.yaml`. During the test script loading process, the values defined in these configuration files are added to the global variables and are then accessible from the test items and scripts (cf. :ref:`global variables`). -The parameter file can be specified in the .tum file root: +The parameter file can be specified in the `.tum` file root: .. code-block:: yaml - :caption: configuration files definition + :caption: configuration files definition in the main `.tum` test file config_file: - - myparam.xml - - config1.json - - config2.yaml + config1.yaml + config2.yaml main: name: Test example [...] -If nothing is specified, the ``param.xml``, ``param.yaml`` and ``param.json`` -are automatically loaded, if present in the test directory. +.. code-block:: yaml + :caption: example of configuration file: param.yaml + + parameter1: value1 + parameter2: 1234 + parameter3: <@ 12.34 * 2 @> + parameter4: + - $(parameter1) + - $(parameter3) + parameter5: + sub_param1: sub_value1 + sub_param2: $(parameter4) + +If nothing is specified, the ``param.yaml`` +is automatically loaded, if present in the test directory. Files loading ^^^^^^^^^^^^^^^^^^ -The ``JSON`` and ``YAML`` configuration files variables are evaluated directly. +The ``YAML`` configuration files variables are evaluated directly and accessible from TUM +tests description files and also from :ref:`python` +and :ref:`lua` function test items. -The XML files content is evaluated as follows. - -.. code-block:: xml - :name: param.xml - - - - - - - - -If the ``parameter`` XML item defines: - -* ``value`` argument: its content is parsed for variable substitution - (see :ref:`variables expansion`) and then evaluated as a python statement, -* ``str`` argument: its content is not evaluated and is kept as a string. +See more details :ref:`below`. .. _sec_global_variables: @@ -153,11 +145,32 @@ library API (see :ref:`helper library`) :ref:`sec_loop_item`). If the loop number its value is the python constant ``inf``. +Debug mode +^^^^^^^^^^^^^^^^^^^ + +Debug mode can be enabled by defining the global variable `test_debug`. + +For example, it can be defined in the configuration file as: + +.. code-block:: yaml + :caption: example of configuration file: param.yaml + + [...] + test_debug: True + [.] + +It can also be defined from the command line with the option +``-d test_debug``. + +When debug mode is enabled, additional information are displayed in the log window. + +Some :ref:`helper library functions` are availabe +to give the state of the debug mode. Test items entries ^^^^^^^^^^^^^^^^^^^^ -All test items attributes can be global variable entry; +All test items attributes can be global variable entries; when using the entry ``$()`` before a key value, the corresponding key entry is searched within the global variables dataset. diff --git a/src/testium/interpreter/utils/jrpc.py b/src/testium/interpreter/utils/jrpc.py index 83d85cf..11cf297 100644 --- a/src/testium/interpreter/utils/jrpc.py +++ b/src/testium/interpreter/utils/jrpc.py @@ -5,6 +5,7 @@ import threading import itertools from time import sleep from typing import Callable, Any +import libs.testium as tm from interpreter.utils.tum_except import ETUMRuntimeError @@ -53,8 +54,14 @@ Notes: class JsonRpcConnection: - def __init__(self, name, conn: socket.socket, req_handler: Callable[..., Any], timeout=0.2, dbg_out=None): - + def __init__( + self, + name, + conn: socket.socket, + req_handler: Callable[..., Any], + timeout=0.2, + dbg_out=None, + ): self.name = name self.conn = conn if not callable(req_handler): @@ -120,9 +127,9 @@ class JsonRpcConnection: def _dispatch(self, msg): if "method" in msg: # request to be sent - meth=msg["method"] - params=msg.get("params", None) - rid=msg.get("id", None) + meth = msg["method"] + params = msg.get("params", None) + rid = msg.get("id", None) threading.Thread( target=self._handle_request, args=(meth, params, rid), daemon=True @@ -168,10 +175,9 @@ class JsonRpcConnection: The send operation is protected by a lock to avoid interleaving when multiple threads attempt to write to the underlying socket. """ - msg = json.dumps(obj) + "\n" data = (msg).encode() - self.print_info("sending : " + msg) + self.print_info(f"sending : " + msg) with self.send_lock: self.conn.sendall(data) @@ -217,6 +223,7 @@ class JsonRpcConnection: def join(self): self.recv_thread.join() + class JsonRpcBase(threading.Thread): """Threaded base class for simple JSON-RPC server/client helpers. @@ -236,7 +243,14 @@ class JsonRpcBase(threading.Thread): - `call()` raises `ETUMRuntimeError` if no active connection exists. """ - def __init__(self, host, port, req_handler: Callable[[dict], Any]=None, timeout=10, dbg_out=None): + def __init__( + self, + host, + port, + req_handler: Callable[[dict], Any] = None, + timeout=10, + dbg_out=None, + ): super().__init__() self._host = host self._port = port @@ -276,7 +290,9 @@ class JsonRpcBase(threading.Thread): self._rpc.stop() def connect(self, sock): - self._rpc = JsonRpcConnection(self.name, sock, self.handle_request, dbg_out=self.dbg_out) + self._rpc = JsonRpcConnection( + self.name, sock, self.handle_request, dbg_out=self.dbg_out + ) self._event_ready.set() def wait_ready(self, timeout=None): @@ -292,6 +308,7 @@ class JsonRpcBase(threading.Thread): if self._rpc is not None: self._rpc.dbg_out = dbg_out + class JsonRpcSrv(JsonRpcBase): """Single-connection JSON-RPC server. @@ -307,7 +324,7 @@ class JsonRpcSrv(JsonRpcBase): The server will raise `ETUMRuntimeError` on accept/connect timeout. """ - def __init__(self, host, port, req_handler = None, timeout=10): + def __init__(self, host, port, req_handler=None, timeout=10): super().__init__(host, port, req_handler, timeout) self.name = f"JsonRpcSvr_{port}" @@ -332,7 +349,6 @@ class JsonRpcSrv(JsonRpcBase): while True: try: conn, addr = sock.accept() - self.print_info("Client connected") break except socket.timeout: t -= tslice @@ -369,11 +385,58 @@ class JsonRpcClient(JsonRpcBase): resp = clt.call('method', {'a': 1}) """ - def __init__(self, host, port, req_handler = None, timeout=10): + def __init__(self, host, port, req_handler=None, timeout=10): super().__init__(host, port, req_handler, timeout) self.name = f"JsonRpcClt_{port}" def run(self): + if tm.OS() == "Windows": + self.run_win() + else: + self.run_lin() + + def run_win(self): + # TCP/IP socket creation + tslice = 1 + t = self._timeout + sock = None + try: + while t >= 0: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(tslice) + # Link of the socket at the configured port + try: + sock.connect((self._host, self._port)) + break + except socket.timeout: + sock.close() + t -= tslice + if t < 0: + raise ETUMRuntimeError( + f"{self.name}: failed to connect : timeout" + ) + else: + sleep(tslice) + except socket.error as e: + raise ETUMRuntimeError(f"{self.name}: failed to connect : {e}") + + self.print_info("Connected to server") + self.connect(sock) + + while self._rpc.running: + # Sleep a short time to avoid a busy loop and allow + # the receiver thread to process messages. + sleep(0.1) + + finally: + if sock is not None: + sock.close() + if self._rpc is not None: + self._rpc.stop() + self._rpc.join() + self.print_info("closed") + + def run_lin(self): # TCP/IP socket creation try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: @@ -385,10 +448,12 @@ class JsonRpcClient(JsonRpcBase): try: sock.connect((self._host, self._port)) break - except OSError as e: + except Exception as e: t -= tslice if t < 0: - raise ETUMRuntimeError(f"{self.name}: failed to connect : {e}") + raise ETUMRuntimeError( + f"{self.name}: failed to connect : {e}" + ) else: sleep(tslice) diff --git a/src/testium/interpreter/utils/lua_func_exec.py b/src/testium/interpreter/utils/lua_func_exec.py index 6629bcf..832d994 100644 --- a/src/testium/interpreter/utils/lua_func_exec.py +++ b/src/testium/interpreter/utils/lua_func_exec.py @@ -1,4 +1,5 @@ import os +import sys import shutil import subprocess import socket @@ -12,12 +13,36 @@ function_call_process = None def lua_func_call_init(lua_path, request_handler, timeout): + """ + Initializes the global Lua function execution process. + + Args: + lua_path (str): Path to the Lua interpreter executable. If empty, uses system default. + request_handler: Handler for JSON-RPC requests. + timeout (int): Timeout for operations in seconds. + + Returns: + LuaFuncExecEngine: The initialized engine instance. + + Raises: + ETUMRuntimeError: If the Lua path is invalid or no interpreter is found. + """ global function_call_process function_call_process = LuaFuncExecEngine(lua_path, request_handler, timeout) return function_call_process def is_lua_interpreter(path: str, timeout=2) -> bool: + """ + Checks if the given path points to a valid Lua interpreter. + + Args: + path (str): Path to the executable to check. + timeout (int, optional): Timeout for the subprocess in seconds. Defaults to 2. + + Returns: + bool: True if the path is a Lua interpreter, False otherwise. + """ try: result = subprocess.run( [path, "-v"], @@ -32,8 +57,25 @@ def is_lua_interpreter(path: str, timeout=2) -> bool: class LuaFuncExecEngine: + """ + Engine for executing Lua functions via a subprocess and JSON-RPC communication. + + This class manages a Lua interpreter subprocess, handles RPC communication, + and executes specified functions with parameters. + """ def __init__(self, lua_path="", request_handler=None, timeout=10): + """ + Initializes the Lua function execution engine. + + Args: + lua_path (str, optional): Path to the Lua interpreter. Defaults to system path. + request_handler: Handler for JSON-RPC requests. + timeout (int, optional): Timeout for operations in seconds. Defaults to 10. + + Raises: + ETUMRuntimeError: If the Lua path is invalid or no interpreter is found. + """ if lua_path != "": if shutil.which(lua_path) is None: raise ETUMRuntimeError( @@ -61,27 +103,49 @@ class LuaFuncExecEngine: def start(self): """ - run the subprocess to execute the python functions of the test. + Starts the Lua subprocess for function execution. + + Sets up environment variables, binds a socket for communication, + and initializes the JSON-RPC client. + + Raises: + ETUMRuntimeError: If the subprocess is already started. """ # This thread is not closed until new test is loaded if self._process is not None: raise ETUMRuntimeError("The function subprocess has already been started.") + func_proc_path = os.path.join(tm.gd("testium_path"),"lua_func") + + # POpen config + CUST_ENV = { + "PATH": {"replace": False}, + "LUA_PATH": {"replace": True}, + "LUA_CPATH": {"replace": True}, + } + + lua_env = tm.gd("lua_env", {}) + env = os.environ.copy() + for k, v in CUST_ENV.items(): + e = lua_env.get(k, "") + if e != "": + if v["replace"]: + env[k] = e + else: + env[k] = e + ";" + env.get(k, "") + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(("localhost", 0)) self._port = sock.getsockname()[1] - func_proc_path = os.path.join(tm.gd("testium_path"),"lua_func") - lua_env = tm.gd("lua_env", {}) - tm.print_debug(f"lua_env : {lua_env}") - + # POpen params params = [self._lpath, "main.lua", "--timeout", f"{self._timeout}", "--host", "127.0.0.1", "--port", f"{self._port}"] - if tm.debug_enabled(): + if tm.debug_enabled() and tm.gd("debug_rpc", False): params.append("--verbose") self._process = subprocess.Popen( - params, env=lua_env, cwd=func_proc_path + params, env=env, cwd=func_proc_path ) # Port was reserved until the sub-process is started. Now released. @@ -89,24 +153,57 @@ class LuaFuncExecEngine: sock.close() self._rpc = JsonRpcClient("localhost", self._port, req_handler=self._req_handler) + if tm.debug_enabled(): + self._rpc.dbg_out = sys.stdout self._rpc.start() def join(self): + """ + Joins the RPC thread and resets the process state. + """ if self._rpc is not None: self._rpc.join() self._rpc = None self._process = None def wait_ready(self, timeout=None): + """ + Waits for the RPC client to be ready. + + Args: + timeout (float, optional): Timeout in seconds. Defaults to None. + + Returns: + bool: True if ready, False otherwise. + """ if self._rpc is not None and self._rpc.is_alive(): return self._rpc.wait_ready(timeout) return False def stop(self): + """ + Stops the RPC client. + """ if self._rpc is not None: self._rpc.stop() def func_call(self, file: str, func_name: str, params: list, verbose: bool = True): + """ + Calls a Lua function via RPC and returns the result. + + Args: + file (str): Path to the Lua file containing the function. + func_name (str): Name of the function to call. + params (list): List of parameters to pass to the function. + verbose (bool, optional): Whether to enable verbose output. Defaults to True. + + Returns: + tuple: (TestValue.SUCCESS, (result, reported_values)) on success, + (TestValue.FAILURE, error_message) on failure. + + Raises: + ETUMRuntimeError: If the RPC call fails or no process is active. + """ if (self._rpc is not None) and self._rpc.is_alive(): answer = self._rpc.call( "func_call", @@ -143,7 +240,21 @@ class LuaFuncExecEngine: def lua_func_exec(file: str, func_name: str, params: list, verbose: bool = True): - """Executes a python function and returns its result and reported values""" + """ + Executes a Lua function using the global function call process. + + Args: + file (str): Path to the Lua file containing the function. + func_name (str): Name of the function to call. + params (list): List of parameters to pass to the function. + verbose (bool, optional): Whether to enable verbose output. Defaults to True. + + Returns: + tuple: (success_status, result_or_error) where success_status is TestValue.SUCCESS or FAILURE. + + Raises: + ETUMRuntimeError: If no function execution process is active. + """ global function_call_process if function_call_process is not None: diff --git a/src/testium/interpreter/utils/paths.py b/src/testium/interpreter/utils/paths.py index 2e16953..b2e6f2c 100644 --- a/src/testium/interpreter/utils/paths.py +++ b/src/testium/interpreter/utils/paths.py @@ -119,7 +119,6 @@ def _sys_app_path_win(app_name): except (FileNotFoundError, PermissionError, subprocess.TimeoutExpired): data = "" sys_python_path = data.splitlines() - tm.print_debug("data = ", data) for l in sys_python_path: if f"{app_name}.exe" in l: return l diff --git a/src/testium/interpreter/utils/py_func_exec.py b/src/testium/interpreter/utils/py_func_exec.py index 8aa4854..2ae50a2 100644 --- a/src/testium/interpreter/utils/py_func_exec.py +++ b/src/testium/interpreter/utils/py_func_exec.py @@ -76,7 +76,8 @@ class PyFuncExecEngine: func_proc_path = tm.gd("testium_path") params = [self._ppath, "-m", "py_func", "-p", f"{self._port}", "-t", f"{self._timeout}"] - if tm.debug_enabled(): + + if tm.debug_enabled() and tm.gd("debug_rpc", False): params.append("-v") self._process = subprocess.Popen( @@ -88,6 +89,8 @@ class PyFuncExecEngine: sock.close() self._rpc = JsonRpcClient("localhost", self._port, req_handler=self._req_handler) + if tm.debug_enabled(): + self._rpc.dbg_out = sys.stdout self._rpc.start() def join(self):