v-and-v/testium
2
0
Fork 0
Code Issues 5 Pull Requests 1 Actions Packages Projects Releases 7 Wiki Activity

Project restart

Browse Source
This commit is contained in:
Foué
2025-12-29 10:46:05 +01:00
commit 59d19cb48c
388 changed files with 48020 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
doc/examples/dummy/cpt/endurance.tum Normal file
View File

@@ -0,0 +1,4 @@
- unittest_file:
name: Test 5
test_file: dummy.py

16
doc/examples/dummy/cpt/temperature.tum Normal file
View File

@@ -0,0 +1,16 @@
sequence: &endurance_test
!include endurance.tum
sequence:
- unittest_file:
name: Test 3
test_file: dummy.py
test_method: test_01_pass
- loop:
iterator: 10
steps:
*endurance_test
- unittest_file:
name: Test 4
test_file: dummy.py

10
doc/examples/dummy/cycle_temperature.py Normal file
View File

@@ -0,0 +1,10 @@
def setTemperature(param):
print('Tempe set : %s'%param)
def temperatureAtteinte(param):
if int(param) > 50:
return True
else:
return False

40
doc/examples/dummy/dummy.py Normal file
View File

@@ -0,0 +1,40 @@
import unittest
from time import sleep
def donothing():
return 0
class DummyTests(unittest.TestCase):
@unittest.skip("test skipped")
def test_00_skipped(self):
''' Test 00 is skipped
'''
sleep(0.5)
def test_01_pass(self):
''' Test 01 is passed and adds a report key
'''
self.reported_values['key reported']= 'value_reported'
sleep(0.5)
def test_02_pass(self):
''' Test 02 is passed and adds a report key
'''
self.reported_values['key reported']= 'toto'
sleep(0.5)
def test_03_fail(self):
''' Test 03 is fail by unittest method
'''
sleep(0.5)
self.fail(msg='Fail message')
def test_04_disabled(self):
''' Test 04 is disabled
'''
sleep(0.5)
def test_05_crash(self):
''' Test 05 crashes
'''
print(crash)

51
doc/examples/dummy/dummy.tum Normal file
View File

@@ -0,0 +1,51 @@
sequence: &temperature_step_sequence
!include tum/temperature.tum
main:
name: Test Sample number one
version: 0.1
steps:
- dialog_references:
name: Ask for reference
question: Please give the reference of the product
reference:
- $(reference_1)
- $(reference_2)
report_show_success: true
- unittest_file:
name: Test 1
test_file: dummy.py
doc: |
Ceci est le test n°1
Voilà...
- sleep:
{name: Sleep between one and two, timeout: 10, dialog: true}
- unittest_file:
{name: Test 2, test_file: dummy.py,execute_on_stop: true}
- loop:
name: Cycle Temperature
iterator : [10,20]
steps:
- py_func:
name: set temperature
file: cycle_temperature.py
func_name: setTemperature
param : $(loop_param)
- *temperature_step_sequence
- py_func:
name: temperature reached
file: cycle_temperature.py
func_name: temperatureAtteinte
param : $(loop_param)
####### REPORT ######
report:
module: test_report_text.py
class: TestReportTxt
title: My Dummy Tests
headers: False
banners: False

4
doc/examples/dummy/param_func.xml Normal file
View File

@@ -0,0 +1,4 @@
<?xml version="1.0" ?>
<root>
<parameter name="func_param" value="param"/>
</root>

13
doc/examples/dummy/plot.py Normal file
View File

@@ -0,0 +1,13 @@
import random
import libs.testium as tm
from libs.testium import FunctionItem
def random_value():
return random.random()
class LastValues(FunctionItem):
def exec(self, name):
res = tm.last_plot_value(name)
self.reportValue('my_reported_value', res)
print("Last plot value: {}".format(res))
return res

10
doc/examples/dummy/post_execution.py Normal file
View File

@@ -0,0 +1,10 @@
import libs.testium as tm
def post_exec():
print('Success !!!!')
print(str(tm.gd('test_outputs')))
def post_exec_fail():
print('Failure :(')
print(str(tm.gd('test_outputs')))

18
doc/examples/dummy/unittest_str.py Normal file
View File

@@ -0,0 +1,18 @@
import unittest
class TestStringMethods(unittest.TestCase):
def test_upper(self):
self.assertEqual('foo'.upper(), 'FOO')
def test_isupper(self):
self.assertTrue('FOO'.isupper())
self.assertFalse('Foo'.isupper())
def test_split(self):
s = 'hello world'
self.assertEqual(s.split(), ['hello', 'world'])
# check that s.split fails when the separator is not a string
with self.assertRaises(TypeError):
s.split(2)

110
doc/examples/example_conditional.tum Normal file
View File

@@ -0,0 +1,110 @@
main:
name: Test conditionals
version: 0.1
stop_on_failure: False
steps:
- group:
name: Set test variables depending on the OS
steps:
- let:
name: Set test variables for Linux
condition: "'$(os)' == 'Linux'"
values:
- terminal_prompt: $(linux_prompt)
- let:
name: Set test variables for Windows
condition: "'$(os)' == 'Windows'"
values:
- terminal_prompt: $(windows_prompt)
# This loop illustrate the way to exit the loop upon operator answer
- loop:
doc: This loop illustrate the way to exit the loop upon operator answer.
name: Infine loop with conditional exit
stop_on_failure: False
steps:
- sleep:
name: sleep item
timeout: 5
- dialog_question:
name: Question to continue
question: Answer yes to exit the loop
no_fail: True
exit_condition:
value: "'$(last_test_result)' == 'PASS'"
- let:
name: let
eval:
- conditional_exec: "random.randint(1, 4)"
- console:
name: Console creation
condition: "$(conditional_exec) == 1"
console_name: consname
doc: Opening the console
steps:
- open:
protocol: terminal
terminal_path: $(test_directory)
- console:
name: Console read_until with timeout
condition: "$(conditional_exec) == 1"
console_name: consname
steps:
- read_until: {expected: "$(terminal_prompt)", timeout: 10}
- console:
name: Console write
condition: "$(conditional_exec) == 1"
console_name: consname
steps:
- writeln: echo 0
- sleep:
name: sleep item
condition: "$(conditional_exec) == 1"
timeout: 5
- console:
name: Console read_until immediate
condition: "$(conditional_exec) == 1"
console_name: consname
steps:
- read_until: {expected: "0", timeout: 0}
- console:
name: Console read_until immediate (2)
condition: "$(conditional_exec) == 1"
console_name: consname
steps:
- read_until: {expected: "$(terminal_prompt)", timeout: 0}
- console:
name: Console closure
condition: "$(conditional_exec) == 1"
console_name: consname
steps:
- close: consname
- sleep:
name: sleep item
condition: "$(conditional_exec) == 2"
timeout: 5
- dialog_image:
name: dialog image item
condition: "$(conditional_exec) == 3"
question: click ok if you see the image
filename: image.jpg
- dialog_value:
name: dialog_value item
condition: "$(conditional_exec) == 4"
question: enter something and click ok

278
doc/examples/example_cycle.tum Normal file
View File

@@ -0,0 +1,278 @@
#This suite make the simplest call to each item.
#
main:
name: Test Sample number one
version: 0.1
stop_on_failure: False
steps:
# This loop contains a failing step and exits.
- loop:
doc: This loop contains a failing step and exits.
name: Infine loop step fails
key: report-key-2
stop_on_failure: True
steps:
- unittest_file:
name: unittest item
doc: |
The purpose of this unittest test item is to demonstrate
its various features.
test_file: dummy/dummy.py
test_method: test_01_pass
- py_func:
name: function test item
doc: The purpose of this step is to demonstrate func test item
file: utils.py
func_name: funcToBeExecuted
param:
- 123
- py_func:
name: function test item
doc: |
The purpose of this step is to demonstrate func test item
and the report key feature.
key: report-key-1
file: utils.py
func_name: funcToBeExecuted
param:
- 123
- unittest_file:
name: Unittest item
test_file: dummy/dummy.py
test_method:
- test_04_disabled
- test_03_fail
exit_condition:
file: utils.py
func_name: dummy_exit
# This loop contains a crashing function and exits.
- loop:
doc: This loop contains a crashing function and exits.
name: Infinite loop func crashes
stop_on_failure: True
steps:
- py_func:
name: function crash
file: utils.py
func_name: funcToBeExecuted2
param:
- 123
# This function crashes and does not fail because of expected result.
- py_func:
name: function crash but no fail
file: utils.py
func_name: funcToBeExecuted2
expected_result: FAIL
param:
- 123
- report:
name: Intermediate report
doc: |
The purpose of this step is to demonstrate the report test item
and its various export features.
export:
- junit:
path: $(home)/reports/report-key-1.junit
pattern:
- Unittest%
key: report-key-1
- text:
file_name: report-key-1.txt
path: $(home)/reports
key:
- report-key-1
# This loop contains a crashing unittest step and exits.
- loop:
doc: This loop contains a crashing unittest step and exits.
name: Infine loop unittest step crashes
stop_on_failure: True
steps:
- unittest_file:
name: Unittest item
test_file: dummy/dummy.py
test_method:
- test_05_crash
# This loop is an example of simple iterator.
- !include seq_cycle.tum
# This loop is an example of time exit condition.
- loop:
doc: This loop is an example of time exit condition.
name: Elapsed time exit condition
steps:
- py_func:
name: function test item
file: utils.py
func_name: funcToBeExecuted
param:
- $(loop_param)
- sleep:
name: Sleep
timeout: 0.2
dialog: false
exit_condition:
time: 0.2
# This loop is an example of value exit condition.
- loop:
doc: This loop is an example of value exit condition
name: Loop exits on a condition
steps:
- let:
name: Set a variable to the loop index
values:
- variable: $(loop_index)
- sleep:
name: Sleep
timeout: 0.2
dialog: false
exit_condition:
value: "$(variable) >= 10"
# This loop must fail du to an exception in exit condition.
- loop:
doc: This loop must fail du to an exception in exit condition.
name: Loop exit_condition crash
steps:
- py_func:
name: function test item
file: utils.py
func_name: funcToBeExecuted
param:
- 123
exit_condition:
file: utils.py
func_name: exit_exc
# This loops include complex iterator composed by global variables
- loop:
doc: This loop includes a complex iterator composed by global variables
name: Loop complex iterator
iterator:
- $(global_loop_param_txt)
- $(global_loop_param_num)
- $(global_loop_param_list)
steps:
- loop:
doc: Loop which print the iterators
name: Simple loop
iterator: $(loop_param)
steps:
- py_func:
name: function test item
file: utils.py
func_name: funcToBeExecuted
param:
- $(loop_param)
- sleep:
name: Sleep be
timeout: 0.5
dialog: false
# Loop with function having a number element of a list as expected result
- loop:
doc: Loop with function having a number element of a list as expected result
name: num list loop
iterator: $(global_loop_param_num)
steps:
- py_func:
name: function test item
file: utils.py
func_name: funcToBeExecuted
param:
- $(loop_param)
expected_result: ($(global_loop_param_num))[$(loop_index)]
- sleep:
name: Sleep be
timeout: 0.5
dialog: false
# Loop with function having a text element of a list as expected result
- loop:
doc: Loop with function having a text element of a list as expected result
name: text list loop
iterator: $(global_loop_param_txt)
steps:
- py_func:
name: function test item
file: utils.py
func_name: funcToBeExecuted
param:
- $(loop_param)
expected_result: ($(global_loop_param_txt))[$(loop_index)]
- sleep:
name: Sleep be
timeout: 0.5
dialog: false
# Loop with function having a list element of a list as expected result
- loop:
doc: Loop with function having a list element of a list as expected result
name: list of list loop
iterator: $(global_loop_param_list)
steps:
- py_func:
name: function test item
file: utils.py
func_name: funcToBeExecuted
param:
- $(loop_param)
expected_result: ($(global_loop_param_list))[$(loop_index)]
- sleep:
name: Sleep be
key: report-key-3
timeout: 0.5
dialog: false
# This is an infinite loop to be stopped manually
- loop:
doc: This is an infinite loop to be stopped manually
name: Infinite loop
skipped: True
steps:
- unittest_file:
name: Unittest item
test_file: dummy/dummy.py
test_method: test_01_pass
- sleep:
name: Sleep between one and two
timeout: 0.5
dialog: false
report:
enabled: True
# log_stored: True
export:
- sqlite:
file_name: $(test_name).sqlite
path: $(home)/reports
# - junit:
# file_name: $(test_name)_report-key-1.junit
# path: $(home)/reports
# pattern:
# - function%
# - Unittest%
# key: report-key-1
# - text:
# file_name: $(test_name)_report-key-2_3.txt
# path: $(home)/reports
# pattern: function%
# key:
# - report-key-2
# - report-key-3

190
doc/examples/example_items.tum Normal file
View File

@@ -0,0 +1,190 @@
#This suite make the simplest call to each item.
#
config_file:
- param.xml
- param.yaml
- param.json
- $(test_directory)/dummy/param_func.xml
main:
name: Test Sample number one
version: 0.1
steps:
- unittest_file:
name: Unittest item
test_file: dummy/unittest_str.py
doc: Unittest test
- group:
name: Set test variables for Linux
condition: "'$(os)' == 'Linux'"
steps:
- let:
name: Set test variables for Linux
values:
- terminal_prompt: $(linux_prompt)
- group:
name: Set test variables for Windows
condition: "'$(os)' == 'Windows'"
steps:
- let:
name: Set test variables for Windows
values:
- terminal_prompt: $(windows_prompt)
- group:
name: Console test group
steps:
- console:
name: Console creation
console_name: consname
doc: Opening the console
steps:
- open:
protocol: terminal
terminal_path: $(test_directory)
- console:
name: Console read_until with timeout
console_name: consname
steps:
- read_until: {expected: "$(terminal_prompt)", timeout: 10}
- console:
name: Console write
console_name: consname
steps:
- writeln: echo 0
- sleep:
name: sleep item
timeout: 5
- console:
name: Console read_until immediate
console_name: consname
steps:
- read_until: {expected: "0", timeout: 0}
- console:
name: Console read_until immediate (2)
console_name: consname
steps:
- read_until: {expected: "$(terminal_prompt)", timeout: 0}
- console:
name: Console closure
console_name: consname
steps:
- close: consname
- loop:
name: cycle item
iterator : 3
steps:
- unittest_file:
name: Unittest item
test_file: dummy/dummy.py
test_method: test_01_pass
- sleep:
name: sleep item
timeout: 1
- loop:
name: cycle item
iterator : 3
steps:
- unittest_file:
name: Unittest item
test_file: dummy/dummy.py
test_method: test_01_pass
- sleep:
name: sleep item
timeout: 1
- py_func:
name: func item
file: dummy/dummy.py
func_name: donothing
- dialog_choices:
name: Choices
question: Select the items you want
icon: $(test_directory)/../../src/testium/main_win/resources/black/document.png
choices:
- name: choice 1
description: My first choice description
icon: $(test_directory)/../../src/testium/main_win/resources/black/document-save.png
choices:
- name: choice 1.1
description: My choice 1 first subchoice description
icon: $(test_directory)/../../src/testium/main_win/resources/black/Label.png
- name: choice 1.2
description: My choice 1 first subchoice description
- name: choice 2
description: My second choice description
icon: $(test_directory)/../../src/testium/main_win/resources/black/image.png
- name: choice 3
description: My third choice description
icon: $(test_directory)/../../src/testium/main_win/resources/black/image.png
choices:
- name: choice 3.1
description: My choice 3 first subchoice description
- name: choice 3.2
description: My choice 3 second subchoice description
icon: $(test_directory)/../../src/testium/main_win/resources/black/Label.png
- dialog_image:
name: dialog image item
question: click ok if you see the image
filename: image.jpg
- sleep:
name: sleep item
timeout: 3
dialog: true
- git:
name: Testium repo
repo: $(test_directory)
- group:
name: group item
steps:
- sleep:
name: sleep item
timeout: 1
- sleep:
name: sleep item
timeout: 1
- dialog_references:
name: dialog_reference item
question: click ok
# reference:
# - ref 1
# - ref 2/rev
# - ref 3/rev/01
- dialog_value:
name: dialog_value item
question: enter something and click ok
- dialog_message:
name: dialog_message item
question: click ok
- dialog_question:
name: dialog_question item
question: click yes
report:
enabled: True
file_name: $(test_name).rep
path: $(home)/reports
pattern: "Console%"
log_stored: False

87
doc/examples/example_plot.tum Normal file
View File

@@ -0,0 +1,87 @@
config_file:
- param.yaml
main:
name: Example of plot item usage
version: 0.1
steps:
- console:
name: Creation of the log dir
console_name: console
steps:
- open:
protocol: terminal
terminal_path: $(test_directory)
- writeln: mkdir -p "$(plot_log_path)" && echo Fini
- read_until: {expected: Fini, timeout: 5}
- console:
name: Console closure
execute_on_stop: true
console_name: console
steps:
- close:
- plot:
name: Open the plot
plot_name: Mon Plot
steps:
- open:
log_path: $(plot_log_path)
- plot:
name: Add periodic to the plot
plot_name: Mon Plot
steps:
- periodic:
period: 1
file: $(test_directory)/dummy/plot.py
func_name: random_value
eval: '{"periodic": $(result)}'
- sleep:
name: sleep
dialog: true
timeout: 3
- loop:
name: Add of other data in the plot
iterator: 10
steps:
- plot:
name: Add to the plot
plot_name: Mon Plot
steps:
- add:
value1: $(loop_index)
value2: $(loop_index)+2
- sleep:
name: sleep between values
timeout: 1
- py_func:
name: last plot values
file: $(test_directory)/dummy/plot.py
func_name: LastValues
param:
- Mon Plot
- plot:
name: Export
execute_on_stop: True
plot_name: Mon Plot
steps:
- export: $(plot_log_path)/plot_export.pdf
- export: $(plot_log_path)/plot_export.csv
- plot:
name: Close the plot
execute_on_stop: True
plot_name: Mon Plot
steps:
- close:
wait_dialog_exit: True
timeout: 60

11
doc/examples/example_run.tum Normal file
View File

@@ -0,0 +1,11 @@
main:
name: Test run Item
steps:
- run:
name: Execute TUM
tum_file: example_cycle.tum
python_path: python3
testium_path: /home/francois/Projets/testium/src/testium
log_file: $(home)/reports/test.log
report_file: $(home)/reports/test.rep

27
doc/examples/example_sequence.tum Normal file
View File

@@ -0,0 +1,27 @@
#This suite use a sequence as a macro to repeat items
#
sequence: &seq_sleep
!include $(sequence)
main:
name: Test Sample number one
version: 0.1
stop_on_failure: False
steps:
# This loop is an example of value exit condition.
- loop:
doc: This loop is an example of value exit condition
name: Loop exits on a condition
steps:
- let:
name: Set a variable to the loop index
values:
- variable: $(loop_index)
- sleep_timeout: $(loop_index)
- *seq_sleep
exit_condition:
value: "$(variable) >= 3"
- !include {file: seq2.tum, is_dialog: True, sleep_timeout: 12, func_para: truc}

18
doc/examples/example_simple.tum Normal file
View File

@@ -0,0 +1,18 @@
main:
name: Test conditionals
version: 0.1
stop_on_failure: False
steps:
- py_func:
name: function test item
doc: The purpose of this step is to demonstrate func test item
file: utils.py
func_name: funcToBeExecuted
param:
- 123
- sleep:
name: sleep item
dialog: True
timeout: 3600

40
doc/examples/example_ssh.tum Normal file
View File

@@ -0,0 +1,40 @@
main:
name: Test ssh
version: 0.1
stop_on_failure: False
steps:
- console:
name: Console creation
console_name: consname
doc: Opening the console
steps:
- open:
protocol: ssh
ssh_host: "my_hostname"
ssh_user: "francois"
- console:
name: Console read
console_name: consname
doc: Opening the console
steps:
- writeln: rm -f /tmp/toto.txt && echo done
- read_until: {expected: "done", timeout: 10}
- read_until: {expected: "$ ", timeout: 1}
- writeln: touch /tmp/toto.txt && echo done
- read_until: {expected: "done", timeout: 10}
- read_until: {expected: "$ ", timeout: 1}
- writeln: echo "blablabla" >> /tmp/toto.txt && echo done
- read_until: {expected: "done", timeout: 10}
- read_until: {expected: "$ ", timeout: 1}
- writeln: cat /tmp/toto.txt && echo done
- read_until: {expected: "done", timeout: 10}
- read_until: {expected: "$ ", timeout: 1}
- console:
name: Console deletion
console_name: consname
doc: Closing the console
steps:
- close: consname

29
doc/examples/example_text_manip.tum Normal file
View File

@@ -0,0 +1,29 @@
main:
name: Test illustrating string manipulation
version: 0.1
stop_on_failure: False
steps:
- let:
name: Set string value
values:
- rand_text: |
Bus 001 Device 001: ID 1d6b:0002 Linux Foundation 2.0 root hub
Bus 002 Device 001: ID 1d6b:0003 Linux Foundation 3.0 root hub
Bus 002 Device 005: ID 0bda:0487 Realtek Semiconductor Corp. Dell dock
Bus 002 Device 006: ID 0bda:0413 Realtek Semiconductor Corp. Dell dock
Bus 002 Device 007: ID 0bda:8153 Realtek Semiconductor Corp. RTL8153 Gigabit Ethernet Adapter
Bus 003 Device 001: ID 1d6b:0002 Linux Foundation 2.0 root hub
Bus 003 Device 003: ID 04f2:b684 Chicony Electronics Co., Ltd Chicony USB2.0 Camera
Bus 003 Device 004: ID 8087:0029 Intel Corp. AX200 Bluetooth
- text_searched: "0bda:8153"
- let:
name: Extract data
eval:
- text_extract: "[l for l in '''$(rand_text)'''.splitlines() if '$(text_searched)' in l][0]"
- dialog_message:
condition: len('$(text_extract)') > 0
name: dialog value test item
question: Tataaaaa !

BIN
doc/examples/image.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 112 KiB

4
doc/examples/param.json Normal file
View File

@@ -0,0 +1,4 @@
{
"global_loop_param_list": [["one", "two", "three"],[1, 2, 3]]
}

9
doc/examples/param.xml Normal file
View File

@@ -0,0 +1,9 @@
<?xml version="1.0" ?>
<root>
<!-- ****************************************************************************
Test configuration
**************************************************************************** -->
<parameter name="windows_prompt" value=">"/>
<parameter name="linux_prompt" value="$"/>
<parameter name="sequence" value="tm.tum"/>
</root>

10
doc/examples/param.yaml Normal file
View File

@@ -0,0 +1,10 @@
#****************************************************************************
# Test configuration
#**************************************************************************** -->
# loops parameters
global_loop_param_txt: ['one', 'two', 'three']
global_loop_param_num: [1, 2, 3]
# Plot parameters
plot_log_path: /tmp/testium_plot/$(testrun_date)/$(testrun_time)/

7
doc/examples/post_execution.py Normal file
View File

@@ -0,0 +1,7 @@
import py_func.tm as tm
def post_exec():
print("post_exec PASS")
def post_exec_fail():
print("post_exec_fail PASS")

17
doc/examples/seq.tum Normal file
View File

@@ -0,0 +1,17 @@
# This sequence uses a parameter defined at the upper level in the file including this sequence
- sequence:
- sleep:
name: Sleep
timeout: $(sleep_timeout)
dialog: false
- dialog_message:
name: dialog_message item
question: value is $(sleep_timeout)
- py_func:
name: function test item
file: utils.py
func_name: funcToBeExecuted
param:
- $(loop_param)

16
doc/examples/seq2.tum Normal file
View File

@@ -0,0 +1,16 @@
- sleep:
name: Sleep {{ sleep_timeout }} sec
timeout: {{ sleep_timeout }}
dialog: {{ is_dialog }}
- dialog_message:
name: dialog_message item
question: value is {{ sleep_timeout }}
- py_func:
name: function test item
file: utils.py
func_name: funcToBeExecuted
param:
- {{ func_param }}

14
doc/examples/seq_cycle.tum Normal file
View File

@@ -0,0 +1,14 @@
# This loop is an example of simple iterator.
- loop:
doc: This loop is an example of simple iterator.
name: Simple iterator
iterator: 25
steps:
- py_func:
name: function test item
file: utils.py
func_name: funcToBeExecuted
param:
- $(loop_param)
expected_result: $(loop_param)

13
doc/examples/utils.py Normal file
View File

@@ -0,0 +1,13 @@
def dummy_exit(useless1, useless2):
return True
def exit_exc(useless1, useless2):
raise Exception('Crash of exit function')
def funcToBeExecuted (bla):
print(bla)
return bla
def funcToBeExecuted2 (bla):
print(bla)
return blo

20
doc/manual/sphinx/Makefile Normal file
View File

@@ -0,0 +1,20 @@
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line, and also
# from the environment for the first two.
SPHINXOPTS ?=
SPHINXBUILD ?= sphinx-build
SOURCEDIR = source
BUILDDIR = build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

15
doc/manual/sphinx/build_doc.sh Executable file
View File

@@ -0,0 +1,15 @@
#! /bin/env sh
SCRIPT_DIR=$(realpath $( dirname "$0"))
ver_file="$(realpath $SCRIPT_DIR/../../../src/VERSION)"
ver=$(echo "$(cat $ver_file)" | cut -d "_" -f 1)
echo "Version of the manual: $ver"
export APP_VERSION=$ver
rm -r $SCRIPT_DIR/build
mkdir $SCRIPT_DIR/build
make -C $SCRIPT_DIR latexpdf || exit
cp -vf $SCRIPT_DIR/build/latex/testium.pdf $SCRIPT_DIR/../testium_manual.pdf

35
doc/manual/sphinx/make.bat Normal file
View File

@@ -0,0 +1,35 @@
@ECHO OFF
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=source
set BUILDDIR=build
if "%1" == "" goto help
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.https://www.sphinx-doc.org/
exit /b 1
)
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
goto end
:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
:end
popd

128
doc/manual/sphinx/source/cdl_interface.rst Normal file
View File

@@ -0,0 +1,128 @@
Command Line Interface
======================
.. code-block:: text
usage: testium.pyw [-h] [--version] [-b] [-m] [-c CONFIG_FILE [CONFIG_FILE ...]] [-r] [-l LOG_FILE]
[-d DEFINE [DEFINE ...]] [-p REPORT_FILE] [-t {sqlite,json,junit,html,text}]
[-n REPORT_PATTERN [REPORT_PATTERN ...]] [-i INCLUDE_PATH [INCLUDE_PATH ...]] [-o] [-g]
[test_file]
positional arguments:
test_file the test script file
optional arguments:
-h, --help show this help message and exit
--version Returns the version of testium
-b, --batch-execution
Executes the test in batch mode
-m, --terminal Starts terminal mode
-c CONFIG_FILE [CONFIG_FILE ...], --config-file CONFIG_FILE [CONFIG_FILE ...]
-o, --no-color Deactivates stdout colors in batch and terminal mode
Configuration file
-r, --run-and-close Runs the test then closes the application
-l LOG_FILE, --log-file LOG_FILE
log file name
-d DEFINE [DEFINE ...], --define DEFINE [DEFINE ...]
Configuration passed to the executed tests.
-p REPORT_FILE, --report-file REPORT_FILE
report file name
-t {sqlite,json,junit,html,text}, --report-type {sqlite,json,junit,html,text}
report file type
-n REPORT_PATTERN [REPORT_PATTERN ...], --report-pattern REPORT_PATTERN [REPORT_PATTERN ...]
report file pattern
-i INCLUDE_PATH [INCLUDE_PATH ...], --include-path INCLUDE_PATH [INCLUDE_PATH ...]
Python modules search path
-g, --debug GUI debug mode
``-h, --help``
--------------
Returns what's in the previous section.
``-b, --batch-execution``
-------------------------
Executes the test in text mode. No need to have QT installed in that case.
``-m, --terminal``
------------------
Starts a testium interactive console. It allows to run commands and sub-tests manually
in a console.
``-o, --no-color``
------------------
Switch allowing to disable the colored output in terminal or batch modes.
``-c, --config-file``
---------------------
This option allows to provide configuration file(s) from the command line.
The configuration files format and content is detailed in the :ref:`config files<sec_configuration_files>` section.
If this parameter is not given while calling *testium*, the default configuration files will be used.
``-r, --run-and-close``
-----------------------
This parameter makes testium to close immediately after running the ``test_file`` argument passed during its call.
If there is no ``test_file`` argument passed, this option is ignored.
``-l, --log-file``
------------------
Path of the log file where to store the log of the test execution.
Goes in a temporary folder if not provided.
.. _sec_option_define:
``-d, --define``
------------------------------------
Defines one or more variables in the form ``VARIABLE1=value1 VARIABLE2=value2 ..."``.
Then, these variables are available from the test scripts, using the :ref:`global variables<sec_global_variables>`
*testium* feature.
.. _sec_p_param:
``-p, --report-file``
----------------------
Path of the report file, stored during the test execution.
This option is only useful in :ref:`batch mode<sec_batch_mode>`.
``-t, --report-type``
---------------------
This option is used in conjuction with option :ref:`-p<sec_p_param>` and is defining
the type of report to be generated.
Please read the :ref:`reports<sec_reports>` section for more details on
the possible types of report.
``-n, --report-pattern``
-------------------------
This option is used in conjuction with option :ref:`-p<sec_p_param>` and is defining
the report parttern(s) used to filter the report results which will be
included in the report file.
More details in :ref:`reports<sec_reports>` section.
``-i, --include-path``
----------------------
Addtional python paths. These paths are appended to the
`sys.path <https://docs.python.org/3/library/sys.html?highlight=sys%20path#sys.path/>`_ python
variable.
``-g, --debug``
---------------
This option is only usefull while debugging *testium* in ``vscode`` in :ref:`graphical mode<sec_graphical_mode>`.

84
doc/manual/sphinx/source/conf.py Normal file
View File

@@ -0,0 +1,84 @@
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
sys.path.insert(0, os.path.abspath('../../../../src/testium/'))
# -- Project information -----------------------------------------------------
project = "testium"
copyright = "2025, François Dausseur"
author = "François Dausseur"
# The full version, including alpha/beta/rc tags
try:
release = os.environ["APP_VERSION"]
version = release
except:
raise Exception("APP_VERSION not defined in environment !")
# -- General configuration ---------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
master_doc = "index"
extensions = [
"sphinx.ext.duration",
"sphinx.ext.doctest",
"sphinx.ext.autodoc",
"sphinx.ext.autosummary",
'linuxdoc.rstFlatTable',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ["includes.rst", "templates.rst", "other_features.rst", "reports.rst"]
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
# html_theme = "alabaster"
html_theme = "classic"
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ["_static"]
numfig = True
latex_engine = "xelatex"
latex_elements = {
"papersize": "a4paper",
'fontpkg': r'''
\setmainfont{DejaVu Sans}
\setsansfont{DejaVu Sans}
\setmonofont{DejaVu Sans Mono}
'''
}
latex_show_urls = "footnote"
pdf_stylesheets = ["style_code_font_size"]
add_module_names = False # Remove namespaces from class/method signatures

BIN
doc/manual/sphinx/source/doc_illustration.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 207 KiB

75
doc/manual/sphinx/source/helper_lib.rst Normal file
View File

@@ -0,0 +1,75 @@
.. _sec_python_helper_library:
Python helper library
======================
A python library including helper function for python modules called from
testium.
To include the support of this library in a python script, the following
line must be included in the script header:
.. code-block:: python
:caption: testium helper library import
import py_func.tm as tm
.. _sec_global_variables_helpers:
Global variables helper functions
----------------------------------
To manage values in the global variables dataset, the following testium library API
must be used:
.. automodule:: interpreter.utils.globdict
:members: gd, setgd, delgd
:undoc-members:
:no-index:
Console helper functions
------------------------
Every opened console instance is added to a list with the
key ``console_instances`` of the global variables.
The instance is removed from the list on close step of the ``console`` test item.
To manage consoles from within ``py_func`` python functions,
the following testium library API can be used:
.. automodule:: libs.testium
:members: add_console, remove_console, console
:undoc-members:
:no-index:
Plot helper functions
------------------------
Every opened plot window instance is added to a list with the
key ``plot_instances`` of the global variables.
The instance is removed from the list on close step of the ``plot`` test item.
To manage plots from within ``py_func`` python functions,
the following testium library API can be used:
.. automodule:: libs.testium
:members: add_plot, remove_plot, plot, add_plot_values, last_plot_value
:undoc-members:
:no-index:
Other helper functions
------------------------
.. automodule:: libs.testium
:members: OS, get_main_dir, timestamp, timestamp_as_sec
:undoc-members:
:no-index:
Debug mode
------------------------
.. automodule:: libs.testium
:members: debug_enabled, enable_debug, print_debug, print_info, print_warn
:undoc-members:
:no-index:

37
doc/manual/sphinx/source/includes.rst Normal file
View File

@@ -0,0 +1,37 @@
Includes
---------
It is possible to include TUM files from another one by using the ``!include`` tag before the included file.
This feature is a testium specific implementation and is not part of the YAML language,
although it is based on the tagging feature of the language and the customization possibility
offered by the python pyYaml package.
Here is a basic example of file inclusion:
.. code-block:: yaml
:caption: included_file.tum
- test_item:
name: test_2
- test_item:
name: test_3
.. code-block:: yaml
:caption: main.tum
#include with the sub-sequence reference mechanism
sequence: &included_sequence
!include included_file.tum
main:
name: Test example
steps:
- test_item1:
name: test_1
- *included_sequence
#include can also be inserted directly within the steps list
- !include included_file.tum

12
doc/manual/sphinx/source/index.rst Normal file
View File

@@ -0,0 +1,12 @@
testium's User's Manual
=====================================
.. toctree::
:maxdepth: 3
:caption: Contents:
overview.rst
cdl_interface.rst
modes.rst
tum_syntax.rst
helper_lib.rst

45
doc/manual/sphinx/source/modes.rst Normal file
View File

@@ -0,0 +1,45 @@
Modes of operation
====================
.. _sec_graphical_mode:
Graphical mode
---------------
*testium* tool has been initially designed to have Graphical User's interface.
The way to call it is simply by executing the ``testium`` command. It is the normal mode.
.. _sec_batch_mode:
Batch mode
----------
The batch mode allows to execute a test in text mode. In this mode, the test does not start any
graphical interface.
.. code-block:: text
:caption: call a test in batch mode
testium -b test/my_test/main.tum
Terminal mode
-------------
The terminal mode starts *testium* in interactive mode. From this console, some tests and
sequences of tests can be called interactively.
.. code-block:: text
:caption: call a test in terminal mode
$ testium -m
Configuration file loaded: /my/execution/path/param.xml
[...]
================================================================================
====== Test configuration
================================================================================
Test executed with testium : 2.4.0 (binary release)
(testium)~

96
doc/manual/sphinx/source/other_features.rst Normal file
View File

@@ -0,0 +1,96 @@
Test outputs
----------------
A list of test result outputs is automatically updated by *testium*.
This is a member of global variables dataset which key is ``test_outputs``.
This global_dict member contains the log file path and, if configured,
the report path as a list.
Other custom logged files may be added by user updated this global variables entry.
Post execution
------------------
A post execution script can be run for example to copy the output files.
For that, a ``post_execution`` element can be defined in the .tum file.
If the test set execution succeeded the ``post_exec`` function of file_name module is run else the ``post_exec_fail`` is run.
If the post_execution element is not defined, the post_execution.py file in the test directory is used by default if existing.
.. code-block:: yaml
:caption: custom post execution python file
post_execution:
file_name: test_report_text.py
Sub-sequence references
-------------------------
It is possible to alias any part of the TUM description file (typically a sequence of steps to be executed) to be inserted within another sequence.
This feature uses the anchor/alias mechanism of the ``YAML`` `language <https://yaml.org/>`_.
Here is an implementation example of a reference to a sub-sequence in a TUM file:
.. code-block:: yaml
:caption: sub-sequences call
sequence: &temperature_step_sequence
- test_item:
name: test_2
- test_item:
name: test_3
main:
name: Test example
steps:
- test_item1:
name: test_1
- *temperature_step_sequence
.. note::
The entry before the alias (``sequence``: in the example above) is needed
mandatorily by YAML language syntax. Nevertheless, its value is not
used by *testium* and thus can be any value.
Test documentation
--------------------
It is possible to display some explicative text user in the GUI.
The ``doc`` attribute of test items is used for that purpose and is displayed as
a tooltip on the test row.
.. code-block:: yaml
:caption: tests documentation
main:
name: Test example
steps:
- unittest_file:
name: unittest item
doc: |
The purpose of this unittest test item is to demonstrate
its various features.
test_file: dummy/dummy.py
test_method: test_01_pass
See illustration in :numref:`Figure %s<doc-illustration>`.
.. figure:: doc_illustration.png
:name: doc-illustration
Illustration of the ``doc`` attribute effect in the GUI.
Unittest
^^^^^^^^^
For ``unittest_file`` type test items, the python docstring of the test method is used as documentation.

32
doc/manual/sphinx/source/overview.rst Normal file
View File

@@ -0,0 +1,32 @@
Overview
========
*testium* is an automated test framework developed in python by François Dausseur.
This software is developed in python and it implements the Qt6 graphical framework.
It has been developed since 2013 with production and development testing in mind.
It's function is to automate the execution of tests. It can be invoked either as command line terminal application or as a graphical interface application.
Tests reports generation and customization are also in this tool's scope.
Its main features are:
* YAML test description,
* Test configuration files in YAML, JSON or XML,
* Full range of pre-existing Test items,
* Test steps, loops,
* Dynamic variables expansion at test runtime,
* Conditional test step execution,
* Modularity of tests (reusable test sequences),
* etc.
All these features give the ability to the test engineer to perform efficient and robust testings.
.. figure:: testium_snapshot.png
testium
Each test is described with the help of a `YAML <https://yaml.org/>`_ file having .tum as extension.
This file is analyzed and then displayed as a tree in a graphical way in the
GUI (see Figure above).

47
doc/manual/sphinx/source/reports.rst Normal file
View File

@@ -0,0 +1,47 @@
.. _sec_reports:
Reports
---------
If a report is required (in addition to the log), the ``report`` YAML element
must be added at the root of the TUM main test file.
The ``report`` YAML element has the following form:
.. code-block:: yaml
:caption: reports global settings
report:
enabled: True
file_name: $(test_name).rep
path: $(home)/reports
pattern: "Console%"
export: junit
log_stored: False
.. table:: report attributes
:widths: 20, 30, 50
+-----------------+-----------------------+-------------------------------------------+
| Attribute | default value | Description |
+-----------------+-----------------------+-------------------------------------------+
| ``enabled`` | ``True`` | Report activated |
+-----------------+-----------------------+-------------------------------------------+
| ``file_name`` | / | Report file name |
+-----------------+-----------------------+-------------------------------------------+
| ``path`` | ``$(report_path)`` | Report storage path By default, it uses |
| | | the default one set in the |
| | | preferences. |
+-----------------+-----------------------+-------------------------------------------+
| ``pattern`` | / | The pattern in SQL wildachars syntax |
| | | to be applied on test names to |
| | | selected reported tests. |
+-----------------+-----------------------+-------------------------------------------+
| ``export`` | / | The type of export. For exemple junit. |
| | | By default, the sqlite format is |
| | | used to generate reports. |
+-----------------+-----------------------+-------------------------------------------+
| ``log_stored`` | / | Defines if the output log of each |
| | | test is accessible to generate the |
| | | report export. |
+-----------------+-----------------------+-------------------------------------------+

4
doc/manual/sphinx/source/style_code_font_size.yaml Normal file
View File

@@ -0,0 +1,4 @@
styles:
code:
parent: literal
fontSize: 8

51
doc/manual/sphinx/source/templates.rst Normal file
View File

@@ -0,0 +1,51 @@
Templates
---------------------------
*testium* embeds the `jinja2 <hhttps://jinja.palletsprojects.com>`_ template engine. It allows a great customization of the
test files, and enforces reusability of test scripts.
In the main test file
^^^^^^^^^^^^^^^^^^^^^^^
The *testium* main test files are systematically passed through the jinja template engine.
The parameters passed to jinja are all the variables contained into the
:ref:`configuration files<sec_configuration_files>` plus the
:ref:`built-in values<sec_global_variables_builtin>`.
In ``!include`` directive
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Along with the basic inclusion capability, there is the possibility to use file inclusion parameters.
Theses parameters are replacing corresponding keywords in bracket in the included file.
See examples below.
.. code-block:: yaml
:caption: including a template
main:
name: Test example
steps:
- test_item1:
name: test_1
#include can also be inserted directly within the steps list
- !include
file: included_template_file.tum
inclusion_parameter_1: param1
inclusion_parameter_2: param2
.. code-block:: yaml
:caption: included template
- test_item:
name: {{ inclusion_parameter_1 }}
- {{ inclusion_parameter_2 }}:
name: test_3
# The following construction is not allowed and will fail to load:
- test_item:
name: {{ $(inclusion)_parameter_3 }}

12
doc/manual/sphinx/source/test_items/check_test_item.rst Normal file
View File

@@ -0,0 +1,12 @@
**check** test item
============================================================
The ``check`` test item returns the result of a python string evaluation:
.. code-block:: yaml
:caption: example of ``check`` test item usage
- check:
name: check test item example
values:
- '"tictactoe" in "$(my_global_string)"'

136
doc/manual/sphinx/source/test_items/console_test_item.rst Normal file
View File

@@ -0,0 +1,136 @@
.. _sec_console_test_item:
**console** test item
============================================================
The console test item is of the form:
.. code-block:: yaml
:caption: example of ``console`` test item usage
- console:
name: test name in GUI
console_name: console name in dict
steps:
- open:
protocol: telnet
telnet_host: $(target_ip)
telnet_port: $(target_port)
- writeln: reset
- read_until: {expected: U-Boot, timeout: 50}
- write: $(boot_vxworks_1)
- writeln: $(boot_vxworks_2)
- read_until:
expected: U-Boot
timeout: 15
- read_until:
expected: Something that will never occurs
timeout: 5
no_fail: True
mute: True
- close:
Attributes
-----------------------
Beside common test items attributes, console test item has specific attributes:
* ``console_name``: console instance name
* ``write_delay``: optional parameter giving the delay to wait in
milliseconds between each character sent.
* ``steps``: a sequence of actions to be applied to the console, as listed above.
The console test item steps accept the parameters and configurations defined in the next sections.
All the following actions support the ``name`` attribute. The ``name`` is concatenated with
the step type in the *testium* GUI, and recalled in the test log and reports.
``open`` action
-------------------------
The ``open`` action initializes the console with the attributes defined as described below.
The console instance is then added to the ``console_instances`` entry of the global
variables (cf :ref:`global variables<sec_global_variables>`).
Open step accepts the following attribute:
* ``protocol``: Setting of the console protocol, supported protocol are listed
in table below
* Other attributes are dependent of the protocol in used and are listed
in table below
.. table:: console protocols
:widths: 20, 30, 50
+---------------+------------------------+-------------------------------------------+
| **Protocol** | **protocol parameter** | **Description** |
+---------------+------------------------+-------------------------------------------+
|``telnet`` | ``telnet_host`` | hostname of the target. |
| +------------------------+-------------------------------------------+
| | ``telnet_port`` | port of the telnet server of the target. |
+---------------+------------------------+-------------------------------------------+
|``ssh`` | ``ssh_host`` | Hostname or IP address of the target. |
| +------------------------+-------------------------------------------+
| | ``ssh_user`` | port of the telnet server of the target. |
| +------------------------+-------------------------------------------+
| | ``ssh_pwd`` | Password (optional). |
+---------------+------------------------+-------------------------------------------+
|``serial`` | ``serial_port`` | Serial port to the target. |
| +------------------------+-------------------------------------------+
| | ``serial_baudrate`` | Baud rate of the serial connection. |
| +------------------------+-------------------------------------------+
| | ``buffered`` | Optinal boolean parameter. If ``False``, |
| | | it forces the |
| | | console to read directly the device. |
| | | Default: ``True``. |
+---------------+------------------------+-------------------------------------------+
|``rawtcp`` | ``tcp_host`` | hostname of the target. |
| +------------------------+-------------------------------------------+
| | ``tcp_port`` | port of the rawtcp server of the target. |
+---------------+------------------------+-------------------------------------------+
|``terminal`` | ``terminal_path`` | Path of the terminal console. |
+ +------------------------+-------------------------------------------+
| | ``shell`` | Shell to execute in the terminal |
| | | Default: /usr/bin/env bash |
+---------------+------------------------+-------------------------------------------+
* ``log``: is available only for Telnet and Serial console and is a path to a folder or a file, where the log will be stored.
``close`` action
---------------------------
The ``close`` action closes the console devices and removes its instance from
the ``console_instances`` list accessible in the global variables
(cf :ref:`global variables<sec_global_variables>`).
No parameters required for this action.
``write`` action
---------------------------
``write`` action takes as parameter the string to be written on the console.
``writeln`` action
-------------------------
writeln function is similar to the write function except that a '\n' (newline) character is sent at the end of the string to be written.
``read_until`` action
----------------------------
The ``read_until`` action is waiting for a string pattern from the console,
its parameter are listed below
* ``expected``: Character string to wait for
* ``timeout``: Timeout setting for the action (in seconds)
* ``no_fail``: Boolean value (``True`` or ``False``) leading to no error reported
if the expected input is not read
* ``mute``: Boolean value (``True`` or ``False``) does not log any readen data
The text read by the ``read_until`` action is stored in the global
variable named ``cn_<test_name>`` (See :ref:`global variables<sec_global_variables>`
for more detail on accessing global variables from test items and scripts).
In the example above, the global variable ``$(cn_test name in GUI)``
would be created at the end of the step. It would contain the resulting
data of the read.

BIN
doc/manual/sphinx/source/test_items/dialog_choices.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 56 KiB

109
doc/manual/sphinx/source/test_items/dialog_choices_test_item.rst Normal file
View File

@@ -0,0 +1,109 @@
.. _sec_dialog_choices_test_item:
**dialog_choices** test item
============================================================
This test item displays a dialog asking a question and waiting for
a selection to be done among defined list of items.
These selectable items can be passed as a tree.
The :numref:`Figure %s<choices-dialog>` displays an example of this item.
.. figure:: dialog_choices.png
:name: choices-dialog
:figwidth: 50 %
:width: 50 %
:align: center
choices dialog
The item parameters corresponding to :numref:`Figure %s<choices-dialog>`
is shown below.
.. code-block:: yaml
:caption: example of ``choices_dialog`` test item usage
- dialog_choices:
name: Choices
question: Select the items you want
icon: $(test_directory)/document.png
choices:
- name: choice 1
description: My first choice description
icon: $(test_directory)/document-save.png
choices:
- name: choice 1.1
description: My choice 1 first subchoice description
icon: $(test_directory)/Label.png
- name: choice 1.2
description: My choice 1 first subchoice description
- name: choice 2
description: My second choice description
icon: $(test_directory)/image.png
- name: choice 3
description: My third choice description
icon: $(test_directory)/image.png
choices:
- name: choice 3.1
description: My choice 3 first subchoice description
- name: choice 3.2
description: My choice 3 second subchoice description
icon: $(test_directory)/Label.png
Attributes
---------------
The supported attributes of the ``dialog_choices`` test item are:
* ``question``: Question to be displayed in the dialog box.
* ``choices``: List of the choicies presented to the user.
* ``icon``: Optional. Path of the icon used in the
selection tree, for all the items by default.
``Choices`` attribute content
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Each choice element is a dictionary which can have the following attributes
* ``name``: name of the choice to be done.
* ``description``: description of the choice to be done.
* ``icon``: Optional. Path of the icon displayed in the
selection tree in front of the corresponding choice.
* ``choices``: List of sub-choicies presented to the user (recursive).
Feature
------------------
The dialog references test item creates the ``cs_<name of test item>`` entry in the
global dictionary.
In the example above, the global variable name containing the
result of the test item would be ``cs_Choices``, and it would contain an
object of this form:
.. code-block::
:caption: example of result of the ``dialog_choices`` test item
[
{'name': 'choice 1',
'checked': True,
'choices': [ {'name': 'choice 1.1', 'checked': True},
{'name': 'choice 1.2', 'checked': False} ]
},
{ 'name': 'choice 2',
'checked': False},
{ 'name': 'choice 3',
'checked': True,
'choices': [ {'name': 'choice 3.1', 'checked': False},
{'name': 'choice 3.2', 'checked': True} ]
}
]
See :ref:`global variables<sec_global_variables>` for more detail
on how to access to global variables from test items and scripts.

27
doc/manual/sphinx/source/test_items/dialog_image_test_item.rst Normal file
View File

@@ -0,0 +1,27 @@
**dialog_image** test item
============================================================
This test item displays an image within a dialog box.
``dialog_image`` test item has the following description format
.. code-block:: yaml
:caption: example of ``dialog_image`` test item usage
- dialog_image:
name: dialog image test item
question: operator question
filename: imageToBeDisplayed.jpg
Attributes
----------------------
``dialog_image`` has the following specific attributes:
* ``question``: Question to be displayed in the dialog box
* ``filename``: File name of the image to be displayed in the dialog box.
Feature
----------------------
The test returns a ``FAIL`` if the answer is No and ``PASS`` if yes.

24
doc/manual/sphinx/source/test_items/dialog_message_test_item.rst Normal file
View File

@@ -0,0 +1,24 @@
**dialog_message** test item
============================================================
This test item displays a simple dialog asking a question and returning the entered value.
dialog_message test item has the following description format
.. code-block:: yaml
:caption: example of ``dialog_message`` test item usage
- dialog_message:
name: dialog value test item
question: operator question
Attributes
---------------------
``dialog_message`` has the following specific attribute:
* ``question``: Sentence to be displayed in the dialog box
Feature
---------------------
Just display the message.

25
doc/manual/sphinx/source/test_items/dialog_note_test_item.rst Normal file
View File

@@ -0,0 +1,25 @@
**dialog_note** test items
============================================================
This test item displays a simple dialog allowing to enter some text and printing the entered value in logs.
``dialog_note`` test item has the following description format
.. code-block:: yaml
:caption: example of ``dialog_note`` test item usage
- dialog_note:
name: dialog value test item
question: operator question
Attributes
-----------------
``dialog_note`` has the following specific attribute:
* ``question``: Question to be displayed in the dialog box
Feature
--------------------
Prints the entered text in the log.

26
doc/manual/sphinx/source/test_items/dialog_question_test_item.rst Normal file
View File

@@ -0,0 +1,26 @@
**dialog_question** test item
============================================================
This test item displays a simple dialog asking a question and returning
the entered value.
``dialog_question`` test item has the following description format
.. code-block:: yaml
:caption: example of ``dialog_question`` test item usage
- dialog_question:
name: dialog value test item
question: operator question
Attributes
--------------------
``dialog_question`` has the following specific attribute:
* ``question``: Question to be asked in the dialog box
Feature
----------------------
The test returns a ``FAIL`` if the answer is No and ``PASS`` if yes.

BIN
doc/manual/sphinx/source/test_items/dialog_reference.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 17 KiB

54
doc/manual/sphinx/source/test_items/dialog_reference_test_item.rst Normal file
View File

@@ -0,0 +1,54 @@
.. _sec_dialog_references_test_item:
**dialog_references** test item
============================================================
This test item displays a dialog asking a question and waiting for
references of the devices under test.
This test item has the following format:
.. code-block:: yaml
:caption: example of ``dialog_references`` test item usage
- dialog_references:
name: ask for a reference
question: Please give the reference of the product
reference:
- ref 1
- ref 2/rev
The example above displays the dialog box in :numref:`Figure %s<dialog-reference>`.
.. figure:: dialog_reference.png
:name: dialog-reference
:figwidth: 50 %
:width: 50 %
:align: center
dialog reference
Attributes
---------------
All the following attributes are mandatory.
* ``question``: Question to be displayed in the dialog box.
* ``reference``: For each of this parameter in the test correspond to a
row to fill in the dialog.
Every field for a reference can be pre-filled using separating
each filed with an '/' (cf :numref:`Figure %s<dialog-reference>`).
Feature
------------------
The dialog references test item creates the ``tested_items`` entry in the
global_dict global variable. This entry is a list of dictionaries of
this form:
.. code-block:: text
:caption: example of ``tested_items`` global variable result of ``dialog_reference``
test item
[{'reference': 'XXXXX', 'revision': 'YYYYY', 'serial': 'ZZZZZ'}, …]

29
doc/manual/sphinx/source/test_items/dialog_value_test_item.rst Normal file
View File

@@ -0,0 +1,29 @@
.. _sec_dialog_value_test_item:
**dialog_value** test items
============================================================
This test item displays a simple dialog asking a question and returning the entered value.
``dialog_value`` test item has the following description format
.. code-block:: yaml
:caption: example of ``dialog_value`` test item usage
- dialog_value:
name: dialog value test item
question: operator question
Attributes
-------------------
``dialog_value`` has the following specific attribute:
* ``question``: Question to be displayed in the dialog box
* ``default``: default value to place in the dialog form (optional)
Feature
----------------------
The returned value is added in the global variable entry with the key being the
``dialog_value`` test item name.

108
doc/manual/sphinx/source/test_items/func_test_item.rst Normal file
View File

@@ -0,0 +1,108 @@
.. _sec_func_item:
**py_func** test item
============================================================
The ``py_func`` test item is used to execute custom python scripts with the given
input parameters.
There are two modes for executing a ``py_func`` item. The class mode and the function mode.
class py_func item
-------------------------
This is the normal way of calling some custom python code.
A class must be defined and derived from ``FunctionItem`` from the ``libs.testium`` module.
From this class it is possible to define some custom reported values with the following API
* ``reportValue(key, value)``: This ``FunctionItem`` method is adding a value added to the report,
* ``reportedValues()``: This ``FunctionItem`` method is retrieving the current report values.
.. code-block:: python
:caption: ``py_func`` test item implementation example
:name: scriptFunctionItem.py
import py_func.tm as tm
class TestItemFunc(tm.FunctionItem)
def exec(param1, param2, param4, param4):
...
self.reportValue('my_reported_value', reported_value)
print(self.reportedValues())
return 10
The ``exec`` method of the ``FunctionItem`` derived class is executed while running the ``py_func`` test item.
.. code-block:: yaml
:caption: legacy ``py_func`` test item implementation
- py_func:
name: function test item
file: scriptTestFile.py
func_name: TestItemFunc
param:
- 123
- 0.123
- True
- $(global_dict_key)
expected_result: 10
**legacy py_func**
The legacy py_func test item is of the form:
.. code-block:: python
:caption: legacy ``py_func`` python function example
:name: scriptTestFile.py
def dummy_func(param1, param2, param4, param4):
...
return 10
There is no possibility to access the report features in that mode.
.. code-block:: yaml
:caption: corresponding ``py_func`` tum extract
- py_func:
name: function test item
file: scriptTestFile.py
func_name: funcToBeExecuted
param:
- 123
- 0.123
- True
- $(global_dict_key)
expected_result: 10
**Attributes**
Beside common test items attributes, py_func item has specific attribute, some of which being mandatory.
* ``file``: the script file name that contains the function to be executed.
Only python script format is supported.
* ``func_name``: The function name to be executed.
* ``param``: This is a list of parameters that are passed to the function
in the order they are presented in the script. These parameters are not
mandatory and are highly dependent of the function prototype.
.. code-block:: yaml
:caption: ``py_func`` test item example of usage
- py_func:
file: script_name.py
func_name: methodName
param:
- $(my_param)
The result of the function (after eventual post treatment) is stored in the global
variable named ``fn_<func_name>``
(See :ref:`global variables<sec_global_variables>` for more detail
on how to access to global variables from test items and scripts).
In the example above, the global variable ``$(fn_function test item)``
would be created at the end of the item execution. It would contain the resulting
value of the funcToBeExecuted python function.

17
doc/manual/sphinx/source/test_items/git_test_item.rst Normal file
View File

@@ -0,0 +1,17 @@
**git** test item
============================================================
Git test item allows
this item has the following description format
.. code-block:: yaml
:caption: ``git`` test item usage example
- git:
name: git test item
repo: [$(test_directory), "/path_to/another/repo"]
Attributes
-----------------
* ``repo``: a string or list of string path to the root of the git repositery(ies) to follow.

26
doc/manual/sphinx/source/test_items/group_test_item.rst Normal file
View File

@@ -0,0 +1,26 @@
**group** test item
============================================================
This element is of the following form:
.. code-block:: yaml
:caption: ``group`` test item usage example
- group:
name: Group Item
condition: "'$(OS)' == 'Linux'"
steps:
- unittest_file:
test_file: test_prod_rio6_8093.py
test_method:
...
- sleep:
timeout: 10
The ``group`` element is used to manage a sequence of item as a group.
Attributes
--------------------
* The ``steps`` list describes the sequence executed in the group.
It is a list of any of the testium test items,

222
doc/manual/sphinx/source/test_items/items_common_attributes.rst Normal file
View File

@@ -0,0 +1,222 @@
.. _sec_item_common:
Items common attributes
============================================================
All test items have common attributes independently of their types, which are
listed in next table, those are all optional parameters and their default value
if not provided is given in the table as well.
.. table:: test items common attributes
:widths: 25, 15, 60
+-----------------------+-------------------+-------------------------------------------------------+
| **Parameter name** | **Default value** | **Description** |
+-----------------------+-------------------+-------------------------------------------------------+
|``name`` | test item type | This is the test item name as displayed |
| | | in the test tree window of the |
| | | testium. |
| | | |
| | | This attribute is also supported by actions of |
| | | ``console``, ``jsonrpc`` or ``plot`` test items. |
| | | |
| | | Default value, if not provided, is the |
| | | test item type. |
+-----------------------+-------------------+-------------------------------------------------------+
|``stop_on_failure`` |``False`` | If ``stop_on_failure`` is set to |
| | | ``True``, the test sequence execution |
| | | stops on test tem failure and no |
| | | further test items are executed, |
| | | except those withexecute_on_stop |
| | | attribute set (see below) |
| | | |
| | | It depends on the test item to take it |
| | | into account or not. |
| | | For example it makes sense to use it |
| | | for ``unittest_file`` test type |
| | | because it can contain many sub-tests, |
| | | but not for sleep test type. |
| | | In cycles, it means that the child |
| | | sequence execution is stopped at first |
| | | failure. It also means that the |
| | | remaining loops are not executed. |
+-----------------------+-------------------+-------------------------------------------------------+
|``execute_on_stop`` |``False`` | When this attribute is set to True, the |
| | | test item is always run, even on test |
| | | failure of any test before. |
| | | This feature is useful, to end the |
| | | test |
| | | sequence properly on test failure |
| | | (switch off power supplies, climatic |
| | | chamber temperature set to ambient |
| | | temperature….) |
+-----------------------+-------------------+-------------------------------------------------------+
|``skipped`` |``False`` | The test item execution is to be |
| | | skipped during test sequence execution |
| | | if set to ``True``. It will be |
| | | displayed as failed in the report. |
+-----------------------+-------------------+-------------------------------------------------------+
| ``no_fail`` | ``False`` | The result of the test step is forced to PASS if this |
| | | attribute is set to ``true``. |
+-----------------------+-------------------+-------------------------------------------------------+
|``doc`` |``""`` | Documentation for the test item that |
| | | appears in the test doc field and the |
| | | contextual text window in the testium |
| | | GUI. |
+-----------------------+-------------------+-------------------------------------------------------+
|``Key`` | / | This attribute defines a key which |
| | | will be attached to the test result and |
| | | which will allow to be filtered during |
| | | the report generation. |
+-----------------------+-------------------+-------------------------------------------------------+
|``report`` | / | This attribute defines values (a dictionary) which |
| | | will be added in the ``data`` field of the report. |
+-----------------------+-------------------+-------------------------------------------------------+
| ``condition`` | / | The test item is not executed if its |
| | | ``condition`` attribute content is |
| | | evaluated as ``False``. |
| | | see :ref:`Conditional |
| | | execution<sec_conditional_execution>`. |
+-----------------------+-------------------+-------------------------------------------------------+
| ``process_result`` | / | Process an evaluation of the process_result |
| | | and store it in the result |
| | | see :ref:`Process result<sec_process_result>` |
| | | for details. |
+-----------------------+-------------------+-------------------------------------------------------+
| ``expected_result`` | / | Expected result value or string. |
| | | see :ref:`Expected result<sec_expected_result>` |
| | | for details. |
+-----------------------+-------------------+-------------------------------------------------------+
last test result
-----------------------------------------------
The global variable ``last_test_result`` is automatically set at the end of a test item execution.
If the corresponding test item does not return any acutal, the content of the ``last_test_result``
variable will be the test success (``PASS``, ``FAIL`` or ``SKIP``).
It the test item returns a value, the ``last_test_result`` variable will contain the returned value.
The main test items returning a value are:
* :ref:`console<sec_console_test_item>` test item,
* :ref:`jsonrpc<sec_jsonrpc_test_item>` test item,
* :ref:`dialog references<sec_dialog_references_test_item>` test item,
* :ref:`dialog value<sec_dialog_value_test_item>` test item.
Test timings
-----------------------------------------------
After the execution of a test step, the following global variables are set :
* ``ts_start_<item_name>``
* ``ts_end_<item_name>``
and
* duration: ``ts_duration_<item_name>``
See :ref:`global variables<sec_global_variables>` for more detail on how to access to global variables from
test items and scripts.
Skipped test items
-----------------------------------------------
A variable named ``skipped_test_item`` can be defined in the global variable entries
or in configuration file (see :ref:`config files<sec_configuration_files>`) as a list of item to be skipped.
.. _sec_conditional_execution:
Conditional execution
-----------------------------------------------
The ``condition`` attribute content is evaluated as a python string.
.. _sec_process_result:
Process result
-----------------------------------------------
The ``process_result`` attribute can be applied to all the test items. However, it's behavior is different
depending if the test item is returning a value or not.
The ``process_result`` attribute content is evaluated as a python line.
The special ``$(result)`` variable is replaced in the ``process_result`` attribute content with the test result value.
The process result is done before the ``expected_result``
If the result of the evaluation is a boolean, the test will be *PASSED* if ``True``, and *FAIL* otherwise.
.. _sec_expected_result:
Expected result
-----------------------------------------------
The ``expected_result`` attribute can be applied to all the test items. However, it's behavior is different
depending if the test item is returning a value or not.
The test items returning a value are:
* :ref:`dialog_references test item<sec_dialog_references_test_item>`
* :ref:`dialog_value test items<sec_dialog_value_test_item>`
* :ref:`py_func test item<sec_func_item>`
* :ref:`dialog_choices test item<sec_dialog_choices_test_item>`
* :ref:`json_rpc test item<sec_jsonrpc_test_item>`
For test items which don't return a value, the ``expected_result`` attribute content is
compared to ``PASS`` or ``FAIL``.
The ``expected_result`` attribute content is a simple comparison with ``$(result)``.
If the result and the expected_result is equal, the test will be *PASSED* if ``True``, and *FAIL* otherwise.
The special ``$(result)`` variable is replaced in the ``expected_result`` attribute content with the test result value.
Export attribute
-----------------------------------------------
.. code-block:: yaml
:caption: Example of ``export`` common attribute usage
- check:
name: Example of result specific to the step 001
values:
- $(last_test_result) == PASS
key:
- GID-1510554_step_1
report:
reported_list: <@ random.sample(range(0,20), k=10) @>
reported_float: <@ math.sqrt(float(1)) @>
reported_str: This is my reported sentence
.. _sec_item_default_folded:
Container items GUI default folding
============================================================
The container items are items which are the parent of other test items. For example loops and groups
are container test items.
In the GUI, if the user wants that a container test item is folded when he opens a test, the ``.``
character has to be place before the test item declaration.
See an example below:
.. code-block:: yaml
:caption: example of ``loop`` folded by default in the GUI
- .loop:
doc: An example loop
name: An example loop
...

178
doc/manual/sphinx/source/test_items/json-rpc_test_item.rst Normal file
View File

@@ -0,0 +1,178 @@
.. _sec_jsonrpc_test_item:
**jsonrpc** test item
============================================================
The `jsonrpc` test item is used to access jsonrpc servers, by sending queries and analysing the
answers.
It supports JSONRPC `v1.0 <https://www.jsonrpc.org/specification_v1>`_ or
`v2.0 <https://www.jsonrpc.org/specification>`_.
This test item can access the jsonrpc server by using an existing
:ref:`console<sec_console_test_item>` or directly using a UDP protocol.
Two low level *adapters* can be then chosen: ``udp`` or ``console``.
Example of ``jsonrpc`` test item with the console adapter:
.. code-block:: yaml
:caption: ``json_rpc`` test item usage example
- json_rpc:
name: JSONRPC console Query
doc: JSONRPC console Query not waiting (only send)
console:
name : jsonrpc_server
prompt: "@@>"
timeout: 1
version: "2.0"
steps:
- query:
method: echo
params:
a: Hello world
b: [0, 1, 2, 3]
id: 3095372
no_wait: True
- [...]
- json_rpc:
name: JSONRPC console Reception
doc: JSONRPC console reception of the previous request
console: {name : jsonrpc_server}
timeout: 1
steps:
- receive:
name: console reception
id: 3095372
timeout: 0.5
Attributes
-----------------------
the jsonrpc attributes are:
* ``timeout``: global communication timeout in seconds. It is a floating point number.
* ``version``: "1.0" or "2.0" (as a string) depending on the version of the JSONRPC
standard which is supported.
* ``mute``: a boolean giving the verbosity of the jsonrpc exchanges on the log output.
* An :ref:`Adapter<sec_jsonrpc_adapters>` is to be chosen between:
* Console,
* UDP,
* ``steps``: a sequence of actions as described in the sections below.
.. _sec_jsonrpc_adapters:
Steps
-----------------------
the jsonrpc steps can be of the following:
* ``open``: used by UDP to open the socket explicitely,
* ``close``: used by UDP adapter to close the socket explicitely,
* ``query``: performs a complete or partial JSONRPC call,
* ``receive``: used to receive the JSONRPC result of call previously
done by the ``query`` action.
If no ``expected_value`` attribute is defined for ``query`` or ``receive`` actions,
the success of the step will depend on the value returned by the JSONRPC frame.
Indeed, this protocol defines a mean to notify if the remote procedure has succeeded
or failed.
All the actions support the ``name`` attribute. The ``name`` is concatenated with
the action type in the *testium* GUI, and recalled in the test log and reports.
adapter attributes
^^^^^^^^^^^^^^^^^^^^^^
The adapters attributes are listed in the table below.
.. flat-table:: jsonrpc adapters
:header-rows: 2
:stub-columns: 1
:widths: 10 20 15 10 10 10
* - :rspan:`1` adapter
- :cspan:`2` attribute
- :cspan:`1` Description
* - attribute
- *type*
* - :rspan:`2` Console
- ``console``
- *dictionary*
- The console adapter configuration
* - ``console.name``
- *string*
- The name of the console which will be retrieved from
the :ref:`global variables<sec_global_variables>`. See also
the :ref:`console test item<sec_console_test_item>`.
* - ``console.prompt``
- *string*
- the eventual enclosing suffix of the jsonrpc frame.
* - :rspan:`4` UDP
- ``udp``
- *dictionary*
- The UDP adapter configuration
* - ``udp.server``
- *string*
- UDP server hostname or IP address.
* - ``udp.snd_port``
- *integer*
- UDP server listening port
* - ``udp.rcv_port``
- *integer*
- UDP answer reception port (on client side)
* - ``bufsize``
- *integer*
- the maximum expected size of the buffer received while waiting for
a jsonrpc frame.
``open`` action
-------------------------
The ``open`` jsonrpc action is only used with the
`UDP adapter<sec_jsonrpc_adapters>` but is mandatory before any ``query`` action.
No parameter is required.
``close`` action
---------------------------
The ``close`` jsonrpc action is only used with the
`UDP adapter<sec_jsonrpc_adapters>` but is mandatory after JSONRPC transfers are finished.
No parameter is required.
``query`` action
---------------------------
The ``query`` jsonrpc action has the following attributes:
* ``method``: JSONRPC method to be called,
* ``params``: JSONRPC param (must be conforming to the version defined above), by default it is an empty list.
* ``id``: JSONRPC id. If not defined or starts with ``rand``, it is chosen randomly.
Otherwise it must be an integer value,
* ``timeout``: reception timeout in seconds. It is a floating point number.
It is by default the jsonrpc timeout.
* ``no_wait``: Optional boolean. False by default. This attribute defines if
the reception is performed in this step (reception can be done appart, in the
``receive`` action described below),
``receive`` action
---------------------------
The ``receive`` jsonrpc action has the following attributes:
* ``id``: JSONRPC id as an integer value,
* ``timeout``: reception timeout in seconds. It is a floating point number,
It is by default the jsonrpc timeout.

25
doc/manual/sphinx/source/test_items/let_test_item.rst Normal file
View File

@@ -0,0 +1,25 @@
**let** test item
============================================================
This element is of the following form:
.. code-block:: yaml
:caption: ``let`` test item usage example
- let:
name: Let Item
values:
key1: value1
key2: value2
eval:
key3: $(variable)[$(loop_index)]
The ``let`` element is used to set values in the global directory.
Attributes
----------------
* The values list gives the {<key>, <value>} couples to set in the
global directory,
* The eval list gives the strings to evaluate prior to its storage into
the <key> of global directory.

95
doc/manual/sphinx/source/test_items/loop_test_item.rst Normal file
View File

@@ -0,0 +1,95 @@
.. _sec_loop_item:
**loop** test items
============================================================
This element is of the following form:
.. code-block:: yaml
:caption: ``loop`` test item usage example
- loop:
name: Cycle Temperature
iterator: 10
steps:
- unittest_file:
test_file: test_prod_rio6_8093.py
- py_func:
name: function test item
file: scriptTestFile.py
func_name: funcToBeExecuted
param:
- $(loop_param)
exit_condition:
file: script_name.py
func_name: methodName
The loop element executes repeatedly the ``steps`` sequence of items.
The configuration of the iteration process is done according to the iterator
cycle sub-item. As described later in this chapter the iterator is
configurable per cycle and allows to call a python function at each
cycle loop.
Attributes
----------------
Below are described loop test item specific attributes.
* ``Iterator``: giving the number of loop iteration (see dedicated chapter below).
* ``steps``: describes the sequence executed at each cycle; it is
a list of any of the testium test items.
* ``exit_condition``: allows to exit the loop. If False is returned, loop continues
else, it breaks. exit_condition attributes are:
* ``time``: the loop stops after the time (in minutes) is elapsed (optional)
* ``value``: the loop stops when the content of the value attribute is
evaluated as True (optional)
* ``file``: the loop the script file name that contains a function to be
executed on each loop. Only python script format is supported (optional
if another exit_condition attribute is defined)
* ``func_name``: the function to execute on each loop when the file attribute
is defined. The function referenced by the ``func_name`` attribute must
have two parameters: the current loop iterator value and the report,
even if they are not used. This attribute is mandatory if the file
attribute is defined.
* ``eval``: optional parameter allowing post treatment of the function result.
It is a python evaluable string in which the ``$(result)`` keywork
is replaced by the actual function call result (see exemple below).
.. code-block:: yaml
:caption: ``loop`` exit condition
- loop:
...
exit_condition:
file: script_name.py
func_name: methodName
eval: $(result) < 2
**Iterator**
The iterator attribute can be of the following types:
* An integer giving the cycle loop number,
* A list. The number of elements of the list gives the loop number, and
the list member are the consecutive loop parameters,
* Undefined. Then cycle loops until the exit condition is reached.
**Loop variables**
The following loop variables are automatically defined:
* ``$(loop_param)``: parameter of the loop. It contains the iterator value.
* ``$(loop_index)``: index of the loop, starting with 0 and incremented at each cycle.
* ``$(loop_index_inverse)``: inverse of index of the loop, starting from cycle length
-1 and decremented at each cycle.
* ``$(loop_count)``: loop total iteration number. If the number of loops is undefined
its value is the python ``inf``.
When these variables are found in a parameter, an attribute, etc, a
loop is searched recursively in the test hierarchy. And the variable value
is replaced by the corresponding loop value.
If more than one loop exists in the test item hierarchy, the lowest level
loop iterator is used.

180
doc/manual/sphinx/source/test_items/plot_test_item.rst Normal file
View File

@@ -0,0 +1,180 @@
**plot** test item
============================================================
This test item is used to display runtime values of tests variables or any evolving value in
a independent external window.
The plot window is defined using the ``plot`` test item:
.. code-block:: yaml
:caption: ``plot`` test item usage example
- plot:
name: test name in GUI
plot_name: plot identifier
steps:
- open:
- add:
...
Attributes
----------------------
In addition to common test items attributes, console test item has specific attributes:
* ``plot_name``: plot window instance name.
* ``steps``: a sequence of actions to be applied to the plot window. More than one action can be
executed in a ``plot`` item.
The plot test item can accept the actions described in further sections.
All the following actions support the ``name`` attribute. The ``name`` is concatenated with
the action type in the *testium* GUI, and recalled in the test log and reports.
``open`` action
----------------------
This action initializes and opens the plot window with the corresponding attributes as defined below.
This action accepts one optional ``path`` parameter defining a path where are stored the
plot lines values in csv format.
.. code-block:: yaml
:caption: ``plot`` ``open`` action
- plot:
name: Open the plot window
plot_name: plot identifier
steps:
- open:
name: open the plot
log_path: $(test_directory)/tmp
``close`` action
----------------------
The ``close`` action closes the plot window and removes its from the managed instances of *testium*.
This action does not have mandatory parameters. However, ``close`` optional action parameters are:
* ``wait_dialog_exit``: Boolean value. If set to True, the window
is kept opened until the user closes it manually.
* ``timeout``: Value expressed in seconds. It is active if the ``wait_dialog_exit`` is set to True.
If this parameter is defined, and if not closed manually, the dialog window is kept opened until
the timeout elapses.
.. code-block:: yaml
:caption: ``plot`` ``close`` action
- plot:
name: Closes the plot
plot_name: plot identifier
steps:
- close:
wait_dialog_exit: True
timeout: 600
.. note::
When the ``close`` action is entered, the ``periodic`` plots are stopped.
``add`` action
----------------------
The ``add`` action is used to add a single data to the ``plot`` window.
.. code-block:: yaml
:caption: ``plot`` ``add`` action
- plot:
name: Add to the plot
plot_name: plot identifier
steps:
- add:
name: add value 1 & 2
value1: $(loop_index)
value2: $(loop_index)+2
The parameter of the ``add`` action is a dictionnary of (*key*, *values*) pairs where the *key* is the plot line name and
*value* is the numeric value to add to the plot line.
The *value* content is evaluated as a python statement if not a number, but a string.
``periodic`` action
----------------------
This action allows to specify a python function to be called and which result is used to update the
plot.
``periodic`` plots are updated automatically and don't require further steps in a test sequence, once executed.
``periodic`` action parameters are:
* ``period``: period of the automatic value update.
* ``file``: python file containing the function to call.
* ``func_name``: the name of the python function to be periodicaly called.
* ``eval``: optional parameter allowing post treatment of the function result.
The ``eval`` parameter of the periodic action is a python evaluable string in which the $(result) keywork
is replaced by the actual function call result.
The result of the action must be a dictionnary of (*key*, *values*) pairs where the *key* is the plot line name and
*value* is the numeric value to add to the plot line.
.. code-block:: yaml
:caption: ``plot`` ``periodic`` action
- plot:
name: Add periodic to the plot
plot_name: plot identifier
steps:
- periodic:
period: 1
file: $(test_path)$(psep)plot.py
func_name: random_value
eval: '{"periodic": $(result)}'
``last_value`` action
----------------------
The ``last_value`` action returns the last values added to the plot (periodicaly or not) into
the global variables entries.
``last_value`` action parameters are:
* ``name``: Optional parameter giving the list of measures to be returned. If
it is not defined, all the measures are returned.
.. code-block:: yaml
:caption: ``plot`` ``last_value`` action
- plot:
name: Plot measure_1 value
plot_name: plot identifier
steps:
- last_value:
name: [measure_1]
The result of the action is stored in the global
variable named ``plv_<item_name>`` in the example above, it would be
``$(plv_Plot measure_1 value)``. See :ref:`global variables<sec_global_variables>` for more detail
on how to access to global variables from test items and scripts.
``export`` action
----------------------
The ``export`` action saves the plot window data in various formats to the filesystem.
.. code-block:: yaml
:caption: ``plot`` ``export`` action
- plot:
name: Plot export
plot_name: plot identifier
steps:
- export: $(my_custom_path)/plot_export.pdf
- export: $(my_custom_path)/plot_export.csv
At the time of writing of this documentation, ``.pdf`` and ``.csv`` files are supported.

51
doc/manual/sphinx/source/test_items/report_test_item.rst Normal file
View File

@@ -0,0 +1,51 @@
**report** test item
============================================================
This test item exports a report file.
To have this functionality activated, a ``report section`` must be defined at the root of the test file.
The root report section is described in :ref:`report<sec_reports>` section.
report test item has the following description format
.. code-block:: yaml
:caption: ``report`` test item usage example
- report:
name: Intermediate report
export:
- junit:
path: $(home)/reports/report-key-1.junit
pattern:
- Unittest%
key: report-key-1
- text:
file_name: report-key-1.txt
path: $(home)/reports
key:
- report-key-1Attributes
This item is useful to generate intermediate reports in any format other than ``sqlite``. Nevertheless,
if ``sqlite`` export is defined, It won't generate anything.
Attributes
---------------------
``report`` test item has the following specific attributes:
* ``export``: reports to be exported. It is a list of the reports exports to be executed.
The supported exports are:
* ``junit``
* ``json``
* ``html``
* text
The export sub-attributes (see example above) may contain the following attributes.
* ``path``: path of the report files directory,
* ``filename``: report file name,
* ``Pattern``: list of the patterns (applied on test names) used to select the
tests to exportinto the report,
* ``Key``: list of selected keys which are used to select the tests to export
into the report.

30
doc/manual/sphinx/source/test_items/run_test_item.rst Normal file
View File

@@ -0,0 +1,30 @@
**run** test item
============================================================
This test item executes a new instance of testium.
.. code-block:: yaml
:caption: ``run`` test item usage example
- run:
name: Execute TUM
tum_fime: example_cycle.tum
python_path: python3
testium_path: /home/francois/projets/testium-new-report/testium.pyw
log_file: $(home)/reports/test.log
report_file: $(home)/reports/test.rep
Attributes
---------------------
run test item has the following specific attributes:
* ``tum_fime``: mandatory the path of the file to execute, it can be relative to current execution folder,
* ``param_file`` (optional) the path of the parameter file to use, otherwise default parameter file is used.
* ``python_path`` (optional) the path of a specific python to run your scripts,
* ``testium_path`` (optional) the path of a specific testium to run your scripts,
* ``log_file`` (optional) the path of log file to register, if not provided a file is created with timestamp at the location of TUM file.
* ``report_file`` (optional), the path of report file to create
* ``start_time`` (optional), start time for the script execution, in HH:MM format.
* ``end_time`` (optional), end time for an execution within a time frame, in HH:MM format.
* ``wait_for_exec`` (optional). True or False, wait to be in the execution window defined by start_time and end_time to run the script.

18
doc/manual/sphinx/source/test_items/sleep_test_item.rst Normal file
View File

@@ -0,0 +1,18 @@
**sleep** test item
============================================================
sleep test item has the following description format
.. code-block:: yaml
:caption: ``sleep`` test item example usage
- sleep:
name: sleep test item
timeout: 10
dialog: True
Attributes
---------------
* ``timeout``: sleep duration in second or in relative date format like "2d 5h 31m 3s", which translate into 2 days, 5 hours, 31 minutes and 3 seconds.
* ``dialog``: If set to True, a window showing the remaining time to wait is displayed (optional parameter set to ``False`` by default)

81
doc/manual/sphinx/source/test_items/unittest_file_test_item.rst Normal file
View File

@@ -0,0 +1,81 @@
**unittest_file** test item
============================================================
unittest_file test item allows the execution of unittest test script which
is part of python standard libraries.
The tum file prototype is as followed:
.. code-block:: yaml
:caption: ``unittest_file`` test item usage example
- unittest_file:
name: unitTest test item
test_file: unitTestScript.py
test_method:
- test_1
- test_2
Attributes
------------------
Beside common test items attributes, unittest test item has specific attribute, some of which being mandatory.
* ``test_file``: it is the name (and eventually path) of the unittest file
to be processed.
* ``test_method``: it is an optional unittest_file test sub-item. If one or more
elements are present, the unittest python script file is parsed and only
the corresponding methods are included in the test tree. Otherwise, all
the test methods are included in the test tree.
Access to global variables entries
----------------------------------
``unittest`` file tests instances have access to the testium global variables
by using the :ref:`helper's library<sec_python_helper_library>`.
Report value from unittest
----------------------------------
Value can be added to the test report from unitTest test at runtime.
.. code-block:: python
:caption: example of ``unittest`` test item python function
from unittest import (TestCase)
class DummyTests(TestCase):
def test_01_report(self):
self.reported_values['key reported']= 'value_reported'
Console use example with unittest item
-----------------------------------------
Here is an example how to use the console module from python ``unittest``.
.. code-block:: python
:caption: example of a *testium* console usage from a ``unittest`` python function
from unittest import (TestCase)
import console
class DummyTests(TestCase):
@classmethod
def setUpClass(cls):
cls.consA0= console.TelnetConsole('cons name','192.168.98.123',7001)
cls.consA0.open()
cls.promptA0 = 'test-computer>'
def test_01_console(self):
self.consA0.write('config')
self.assertEqual(self.consA0.read_until(self.promptA0, 10), 0)
self.consA0.write('lsusb && echo "Done."\n')
status, read_data = self.consA0.read_until('Done.',
10, return_data=True)
self.assertEqual(status, 0)
if read_data.find('ID 04f2:b684 Chicony Electronics Co.')!=-1:
index=0
@classmethod
def tearDownClass(cls):
cls.consA0.close()

BIN
doc/manual/sphinx/source/testium_snapshot.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 200 KiB

254
doc/manual/sphinx/source/tum_syntax.rst Normal file
View File

@@ -0,0 +1,254 @@
TUM file syntax
================
*testium* is a python-based tool which uses a ``YAML`` based description file to operate tests: the TUM file.
The description of tests is based on the definition of test sequences. There is a main ``YAML`` element which is the *testium* tool entry point.
All other steps are listed in the step list of the main. Some steps are themselves list of steps as well, such as loop or console items.
``YAML`` is an indented language and parameter encapsulation is defined by their indentation.
``YAML`` language auto detects data type so that it is not necessary to cast the element type explicitly. See `YAML home page <https://yaml.org/>`_ for further information on the language.
The example below shows a basic implementation of the TUM description file:
.. code-block:: yaml
:caption: main test file
main:
name: Test example
steps:
- test_item1:
name: test_1
- loop :
name: test cyle
iterator: 5
steps:
- test_item:
name: test_2
- test_item:
name: test_3
.. _sec_configuration_files:
Configuration files
--------------------
A configuration file can be specified in the .tum file or by the command line.
This configuration file is optional.
It can be of three different syntax:
* XML
* YAML
* JSON
The type of file is recognized by the file name extension (.xml, .yaml, .json).
During the test script loading process, the values defined in these configuration files
are added to the global variables and are then accessible from the test items and scripts
(cf. :ref:`global variables<sec_global_variables>`).
The parameter file can be specified in the .tum file root:
.. code-block:: yaml
:caption: configuration files definition
config_file:
- myparam.xml
- config1.json
- config2.yaml
main:
name: Test example
[...]
If nothing is specified, the ``param.xml``, ``param.yaml`` and ``param.json``
are automatically loaded, if present in the test directory.
Files loading
^^^^^^^^^^^^^^^^^^
The ``JSON`` and ``YAML`` configuration files variables are evaluated directly.
The XML files content is evaluated as follows.
.. code-block:: xml
:name: param.xml
<?xml version="1.0" ?>
<root>
<parameter name="param1" value="['abc', 'bcd']"/>
<parameter name="param2" value="0x123454"/>
<parameter name="param3" str="def"/>
</root>
If the ``parameter`` XML item defines:
* ``value`` argument: its content is parsed for variable substitution
(see :ref:`variables expansion<sec_variable_expansion>`) and then evaluated as a python statement,
* ``str`` argument: its content is not evaluated and is kept as a string.
.. _sec_global_variables:
Global variables
-------------------
Global variables feature is adding the possibility for test items and test scripts to access a common
and global variables database.
The global variables dataset is populated from various sources:
* the :ref:`command line<sec_option_define>`,
* :ref:`built-in values<sec_global_variables_builtin>`,
* the :ref:`configuration files<sec_configuration_files>`,
* some test items results,
* the :ref:`helper library API<sec_global_variables_helpers>`, accessible from python scripts.
Theses global variables are used for variable expansion in scripts (see :ref:`variables expansion<sec_variable_expansion>`).
Another possible usage of the global variables is to share persistent data between test steps.
A library allowing python functions to access global variables is available from the
python scripts. See details in section :ref:`helper library<sec_python_helper_library>`.
Apart from the value obtained from the param.xml file, the global varibles entries
contains also built-in specific value, and test item specific values.
.. _sec_global_variables_builtin:
Built-in values
^^^^^^^^^^^^^^^^^^^
The following keys are automatically accessible through the testium
library API (see :ref:`helper library<sec_python_helper_library>`)
* ``test_directory``: the absolute path of the directory of the main .tum file,
* ``test_main_file``: the main .tum file,
* ``os``: the name of the platform which is used. Can be Linux or Windows,
* ``host_name``: The name of the host on which testium is running,
* ``home``: home directory of the current user,
* ``testrun_date``: The date when the test has started (as a string) in
format YYYY-MM-DD,
* ``testrun_time``: The time when the test has started (as a string) in
format HH:MM:SS,
* ``test_name``: The name of the file being executed without extension,
* ``home``: the path of the current user's home directory,
* ``test_outputs``: list of the paths of the test log and test report (if any),
* ``last_test_result``: test result of the last step (see :ref:`sec_item_common`),
* ``ts_start_<item_name>``: timestamp at the start of test item execution (see :ref:`sec_item_common`),
* ``ts_end_<item_name>``: timestamp at the end of test item execution (see :ref:`sec_item_common`),
* ``ts_duration_<item_name>``: duration of test item execution in seconds (see :ref:`sec_item_common`),
* ``cn_<test_name>``: console test item result (see section :ref:`sec_console_test_item`),
* ``fn_<func_name>``: py_func test item result (see section :ref:`sec_func_item`),
* ``cs_<test_name>``: dialog_choices test item result (see section :ref:`sec_dialog_choices_test_item`),
* ``loop_param``: loop iterator (available from within a loop item,
see :ref:`sec_loop_item`),
* ``loop_index``: loop index (available from within a loop item, see
:ref:`sec_loop_item`),
* ``loop_count``: loop number (available from within a loop item, see
:ref:`sec_loop_item`). If the loop number its value is the python constant
``inf``.
Test items entries
^^^^^^^^^^^^^^^^^^^^
All test items attributes can be global variable entry;
when using the entry ``$(<global>)`` before a key value, the corresponding
key entry is searched within the global variables dataset.
References
^^^^^^^^^^^^^^
If the ``dialog_references`` test item has been included (see
:ref:`dialog_reference test item<sec_dialog_references_test_item>`), the global
dict will contain the result of this test item in the key ``tested_items``.
Dialog values
^^^^^^^^^^^^^^^^
All dialog returned values are inserted in the global variables entries with the
key value being the test item name attribute (see :ref:`dialog_value test item<sec_dialog_value_test_item>`).
.. _sec_variable_expansion:
Paramers passing, variable expansion and evaluations
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
One of the most useful functionalities for scalability and flexibility of the
.tum files is the ability to expanse variables at test runtime.
It is done by replacing any occurrence of ``$(my_global)`` with the content of
the variable in the global variables entries (see :ref:`global variables<sec_global_variables>`).
The variable substitution is recursive and checks all the occurrences of the
``$(x)`` pattern in a string.
It is also possible to perform evaluation of python substrings during parameters passing.
It is done by using the ``<@ expr @>`` pattern in a string.
`expr` may then be a correct python expression.
Below are illustrated simple and more complicated cases of expansion and evaluation depending on
their pattern.
.. code-block:: yaml
:caption: variables expansion and evaluation
- let:
name: Dynamic variables expansion
key: $(test)_PASS
values:
- expanse_select: <@"$(expanse_select)".replace("o", "a")@>
- expanse_index: $(expanse_index_$(expanse_select))
- expanse_table: $(expanse_table_$(expanse_select))
- expanse_eval: <@$(expanse_index) == 1@>
Test Items
--------------------
All *testium* steps are described in sequence as test items in the step
list of the main test item (and eventually of the loop test item).
TUM file ``main`` item is itself a variant of test items with a name and an
step list attributes.
.. toctree::
:maxdepth: 3
:caption: Contents:
test_items/items_common_attributes.rst
test_items/check_test_item.rst
test_items/console_test_item.rst
test_items/dialog_choices_test_item.rst
test_items/dialog_image_test_item.rst
test_items/dialog_message_test_item.rst
test_items/dialog_note_test_item.rst
test_items/dialog_question_test_item.rst
test_items/dialog_reference_test_item.rst
test_items/dialog_value_test_item.rst
test_items/func_test_item.rst
test_items/git_test_item.rst
test_items/group_test_item.rst
test_items/json-rpc_test_item.rst
test_items/let_test_item.rst
test_items/loop_test_item.rst
test_items/plot_test_item.rst
test_items/report_test_item.rst
test_items/run_test_item.rst
test_items/sleep_test_item.rst
test_items/unittest_file_test_item.rst
.. include:: includes.rst
.. include:: templates.rst
.. include:: reports.rst
.. include:: other_features.rst

BIN
doc/manual/testium_manual.pdf Normal file
View File

Binary file not shown.