v-and-v/testium
2
0
Fork 0
Code Issues 5 Pull Requests 1 Actions Packages Projects Releases 7 Wiki Activity

Project restart

Browse Source
This commit is contained in:
Foué
2025-12-29 10:46:05 +01:00
commit 59d19cb48c
388 changed files with 48020 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
test/robustness/console serial/rsl_terminal_robustness.tum Normal file
View File

@@ -0,0 +1,69 @@
# Main
################################################################################
main:
name: Serial Terminal bug reproducer
version: 0.1
steps:
- group:
name: Test preparation
steps:
- console:
name: Open RSL Simulator Terminal
console_name: RSL_simulator
steps:
- open:
protocol: terminal
terminal_path: $(rslsimulatorpath)
- writeln: "pwd"
- read_until: {expected: "$", timeout: 5}
- writeln: "./RSverify $(rsTx)" # /dev/ttyMUE1
- read_until: {expected: "RSL controller>", timeout: 5}
- writeln: "setportconf 0 115200 none 8 1 1 255"
- read_until: {expected: "RSL controller>", timeout: 5}
- writeln: "send4ever 0 0"
- read_until: {expected: "RSL controller>", timeout: 5}
- console:
name: Open the EUT console
console_name: cons_target
doc: Initiates the console of the target in order
to be ready to capture its traces.
stop_on_failure: True
steps:
- open:
protocol: serial
serial_port: $(rsRx) # /dev/ttyMUE2
serial_baudrate: 115200
- loop:
name: Qualification loop
stop_on_failure: False
steps:
- py_func:
name: Capture the RS serial output
file: $(test_directory)/terminal_bug_reproducer.py
func_name: RetreiveData
param:
- cons_target
- sleep: {timeout: 1}
# Cleanup sequence
#-------------------------------------------------------------------------------
- group:
name: Cleanup
execute_on_stop: True
steps:
- console:
name: Close the target console
console_name: cons_target
execute_on_stop: True
steps:
- close:
- console:
name: Close the RSL_simulator
console_name: RSL_simulator
execute_on_stop: True
steps:
- close:

26
test/robustness/console serial/terminal_bug_reproducer.py Normal file
View File

@@ -0,0 +1,26 @@
import libs.testium as tm
def RetreiveData(console_name):
print("--------------- retrieving data ---------------")
result = 0
cons = tm.console(console_name)
if cons is None:
print("--------------- The console does not exist ---------------")
else:
try:
is_finished = False
while not is_finished:
status, d = cons.read_until('\n', timeout=0, return_data=True, mute=True)
if 0 == status:
print("--------------- Data ---------------")
print(d)
else:
print("--------------- No data ---------------")
print("Status: ", status)
is_finished = True
except:
print("--------------- Error retrieving data ---------------")
result = -1
return result

9
test/robustness/console terminal/generate_char.sh Executable file
View File

@@ -0,0 +1,9 @@
chars='<=>| -,;:!/."()[]{}*\&#%+012345689abcdefghiklmnopqrstuvwxyzABCD'
for j in {1..256} ;
do
for i in {1..256} ; do
echo -n "${chars:RANDOM%${#chars}:1}"
done
echo
sleep 0.01
done

26
test/robustness/console terminal/terminal_bug_reproducer.py Normal file
View File

@@ -0,0 +1,26 @@
import libs.testium as tm
def RetreiveData(console_name):
print("--------------- retrieving data ---------------")
result = 0
cons = tm.console(console_name)
if cons is None:
print("--------------- The console does not exist ---------------")
else:
try:
is_finished = False
while not is_finished:
status, d = cons.read_until('\n', timeout=0, return_data=True, mute=True)
if 0 == status:
print("--------------- Data ---------------")
print(d)
else:
print("--------------- No data ---------------")
print("Status: ", status)
is_finished = True
except:
print("--------------- Error retrieving data ---------------")
result = -1
return result

50
test/robustness/console terminal/terminal_robustness.tum Normal file
View File

@@ -0,0 +1,50 @@
# Main
################################################################################
main:
name: Terminal bug reproducer
version: 0.1
steps:
- group:
name: Test preparation
steps:
- console:
name: Open the EUT console
console_name: cons_target
doc: Initiates the console of the target in order
to be ready to capture its traces.
stop_on_failure: True
steps:
- open:
protocol: terminal
- loop:
name: Qualification loop
stop_on_failure: False
steps:
- console:
name: write random data
console_name: cons_target
steps:
- writeln: bash $(test_directory)/generate_char.sh
- py_func:
name: Capture the terminal output
file: $(test_directory)/terminal_bug_reproducer.py
func_name: RetreiveData
param:
- cons_target
- sleep: {timeout: 1}
# Cleanup sequence
#-------------------------------------------------------------------------------
- group:
name: Cleanup
execute_on_stop: True
steps:
- console:
name: Close the target console
console_name: cons_target
execute_on_stop: True
steps:
- close:

0
test/tmp/.gitkeep Normal file
View File

10
test/validation/README.md Normal file
View File

@@ -0,0 +1,10 @@
# Validation
This directory contains the necessary material to run the testium validation.
Here is the documentation on how to configure the validation, run it and check that the
results are correct.
# Tests
TBD

3
test/validation/items/check/check.py Normal file
View File

@@ -0,0 +1,3 @@
def echo(param):
return param

1
test/validation/items/check/param.yaml Normal file
View File

@@ -0,0 +1 @@
no_param: Null

28
test/validation/items/check/test.tum Normal file
View File

@@ -0,0 +1,28 @@
- py_func:
name: Dummy_int
file: $(test_path)$(psep)check.py
func_name: echo
key: $(test)_PASS
param:
- 2
- py_func:
name: Dummy_str
file: $(test_path)$(psep)check.py
func_name: echo
key: $(test)_PASS
param:
- my taylor is rich
- check:
name: Check condition on existing variable (PASS)
key: $(test)_PASS
values:
- $(fn_Dummy_int) > 1
- check:
name: Check condition on existing variable (FAIL)
key: $(test)_FAIL
values:
- '"tailor" in "$(fn_Dummy_str)"'

3
test/validation/items/common/conditional/conditional.py Normal file
View File

@@ -0,0 +1,3 @@
def echo(param):
return param

BIN
test/validation/items/common/conditional/image.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 112 KiB

86
test/validation/items/common/conditional/test.tum Normal file
View File

@@ -0,0 +1,86 @@
# This loop illustrate the way to exit the loop upon operator answer
- loop:
doc: This loop illustrate the way to exit on a condition.
name: Infine loop with conditional exit
stop_on_failure: False
key: $(test)_PASS
steps:
- sleep: {name: small wait, timeout: 0.2}
- py_func:
name: Echo function
key: $(test)_PASS
file: $(test_path)$(psep)conditional$(psep)conditional.py
func_name: echo
param:
- $(loop_param)
exit_condition:
value: "$(fn_Echo function) > 3"
- let:
name: let
key: $(test)_PASS
eval:
- conditional_exec: "random.randint(1, 2)"
- console:
name: Console creation
condition: "$(conditional_exec) == 1"
console_name: consname
doc: Opening the console
key: $(test)_PASS
steps:
- open:
protocol: terminal
terminal_path: $(test_directory)
- console:
name: Console read_until with timeout
condition: "$(conditional_exec) == 1"
console_name: consname
key: $(test)_PASS
steps:
- read_until: {expected: "$(terminal_prompt)", timeout: 10}
- console:
name: Console write
condition: "$(conditional_exec) == 1"
console_name: consname
key: $(test)_PASS
steps:
- writeln: echo 0
- sleep:
name: sleep item
condition: "$(conditional_exec) == 1"
timeout: 1
- console:
name: Console read_until immediate
condition: "$(conditional_exec) == 1"
console_name: consname
key: $(test)_PASS
steps:
- read_until: {expected: "0", timeout: 0}
- console:
name: Console read_until immediate (2)
condition: "$(conditional_exec) == 1"
console_name: consname
key: $(test)_PASS
steps:
- read_until: {expected: "$(terminal_prompt)", timeout: 0}
- console:
name: Console closure
condition: "$(conditional_exec) == 1"
console_name: consname
key: $(test)_PASS
steps:
- close: consname
- sleep:
name: sleep item
condition: "$(conditional_exec) == 2"
timeout: 1

5
test/validation/items/common/param.yaml Normal file
View File

@@ -0,0 +1,5 @@
dict_example: {nbr: 42, msg: "The Answer is 42"}
list_example: [42, msg: "The Answer is 42"]
str_example: "The Answer is 42"
nbr_example: 42
no_param: Null

10
test/validation/items/common/results/results.py Normal file
View File

@@ -0,0 +1,10 @@
def echo(param):
print(param)
return param
def raise_issue(param):
raise RuntimeError(str(param))
def return_none():
return None

155
test/validation/items/common/results/test.tum Normal file
View File

@@ -0,0 +1,155 @@
- group:
name : Expected Result
steps:
- py_func:
name: Return True expect True
key: $(test)_PASS
file: $(test_path)$(psep)results$(psep)results.py
func_name: echo
param: [ True ]
expected_result: True
- py_func:
name: Return True expect False (must fail)
key: $(test)_FAIL
file: $(test_path)$(psep)results$(psep)results.py
func_name: echo
param: [ True ]
expected_result: False
- py_func:
name: Return None expect None
key: $(test)_PASS
file: $(test_path)$(psep)results$(psep)results.py
func_name: return_none
expected_result: None
- py_func:
name: Return None expect PASS
key: $(test)_PASS
file: $(test_path)$(psep)results$(psep)results.py
func_name: return_none
expected_result: PASS
- py_func:
name: Return None expect 14 (must fail)
key: $(test)_FAIL
file: $(test_path)$(psep)results$(psep)results.py
func_name: return_none
expected_result: 14
- group:
name : Expected Result Last test result
steps:
- py_func:
name: result is 28
key: $(test)_PASS
file: $(test_path)$(psep)results$(psep)results.py
func_name: echo
param: [ 28 ]
- py_func:
name: check that the last test result is 28
key: $(test)_PASS
file: $(test_path)$(psep)results$(psep)results.py
func_name: echo
param: [ 28 ]
expected_result: "$(last_test_result)"
- group:
name : Expected result Failure raised issue
steps:
- py_func:
name: Raise an issue (must fail)
key: $(test)_FAIL
file: $(test_path)$(psep)results$(psep)results.py
func_name: raise_issue
param: [ $(str_example) ]
- py_func:
name: Raise an issue and expected the test to be FAIL
key: $(test)_PASS
file: $(test_path)$(psep)results$(psep)results.py
func_name: raise_issue
param: [ $(str_example) ]
expected_result: "FAIL"
- py_func:
name: Return a String expect a FAILURE (must fail)
key: $(test)_FAIL
file: $(test_path)$(psep)results$(psep)results.py
func_name: echo
param: [ $(str_example) ]
expected_result: "FAIL"
- group:
name : process result
steps:
- py_func:
name: Process result equal String
key: $(test)_PASS
file: $(test_path)$(psep)results$(psep)results.py
func_name: echo
param: [ $(str_example) ]
process_result: "'$(str_example)' == '$(result)'"
- py_func:
name: Process result string in the result (must fail)
key: $(test)_FAIL
file: $(test_path)$(psep)results$(psep)results.py
func_name: echo
param: [ $(str_example) ]
process_result: "'44' in '$(result)'"
- py_func:
name: Save the result in a global variable
key: $(test)_PASS
file: $(test_path)$(psep)results$(psep)results.py
func_name: echo
param: [ 44 ]
process_result: "tm.setgd('process_result_value', $(result))"
- py_func:
name: Check the saved global variable
key: $(test)_PASS
file: $(test_path)$(psep)results$(psep)results.py
func_name: echo
param: [ 44 ]
expected_result: $(process_result_value)
- py_func:
name: Process result when result is None (must fail)
key: $(test)_FAIL
file: $(test_path)$(psep)results$(psep)results.py
func_name: return_none
process_result: "$(result) is None"
- group:
name : no_fail result
steps:
- py_func:
name: Return True expect False but no_fail=True
no_fail: True
key: $(test)_PASS
file: $(test_path)$(psep)results$(psep)results.py
func_name: echo
param: [ True ]
expected_result: False
- py_func:
name: Return True expect False but no_fail=False (must fail)
no_fail: False
key: $(test)_FAIL
file: $(test_path)$(psep)results$(psep)results.py
func_name: echo
param: [ True ]
expected_result: False
- py_func:
name: Return True expect False but no_fail expansed
no_fail: <@ bool(0) == False @>
key: $(test)_PASS
file: $(test_path)$(psep)results$(psep)results.py
func_name: echo
param: [ True ]
expected_result: False
- py_func:
name: Return True expect False but no_fail expansed (must fail)
no_fail: <@ bool(1) == False @>
key: $(test)_FAIL
file: $(test_path)$(psep)results$(psep)results.py
func_name: echo
param: [ True ]
expected_result: False

16
test/validation/items/common/syntax_robustness/test.tum Normal file
View File

@@ -0,0 +1,16 @@
# No name declared
- sleep: {timeout: 0.2, key: $(test)_PASS}
# Empty name declared
- sleep:
name:
timeout: 0.2
key: $(test)_PASS
# Empty doc declared
- sleep:
name: Empty "doc:" declared (must PASS)
doc:
timeout: 0.2
key: $(test)_PASS

12
test/validation/items/common/test.tum Normal file
View File

@@ -0,0 +1,12 @@
- group:
name : Results
steps:
- !include results/test.tum
- group:
name : Conditional
steps:
- !include conditional/test.tum
- group:
name : Various syntax robustness
steps:
- !include syntax_robustness/test.tum

1
test/validation/items/console/param.yaml Normal file
View File

@@ -0,0 +1 @@
no_param: Null

88
test/validation/items/console/test.tum Normal file
View File

@@ -0,0 +1,88 @@
- console:
name: Console creation
console_name: term
doc: Opening the console
key: $(test)_PASS
steps:
- open:
protocol: terminal
terminal_path: $(test_directory)
- writeln: echo "endOfOpen"
- console:
name: Console read_until with timeout
console_name: term
key: $(test)_PASS
steps:
- read_until: {expected: endOfOpen, timeout: 5}
- console:
name: Console write
console_name: term
key: $(test)_PASS
steps:
- writeln: echo 0
- sleep:
name: sleep item
timeout: 1
- console:
name: Console read_until immediate
console_name: term
key: $(test)_PASS
steps:
- read_until: {expected: "0", timeout: 0}
- console:
name: Console write
console_name: term
key: $(test)_PASS
steps:
- writeln: echo "HelloConsole"
- console:
name: Console read_until fail
console_name: term
key: $(test)_FAIL
steps:
- read_until: {expected: "Something never prints", timeout: 1}
- console:
name: Console write
console_name: term
key: $(test)_PASS
steps:
- writeln: echo "HelloConsole"
- console:
name: Console read_until no_fail
console_name: term
key: $(test)_PASS
steps:
- read_until: {expected: "Something never prints", timeout: 1, no_fail: true}
- console:
name: Console read_until muted
console_name: term
key: $(test)_PASS
steps:
- writeln: echo "HelloConsole"
- read_until: {expected: "HelloConsole", timeout: 1, mute: true}
- console:
name: Console read_until muted
console_name: term
key: $(test)_PASS
steps:
- writeln: echo "HelloConsole is PASS" && echo "endOfCmd"
- read_until: {expected: endOfCmd, timeout: 1, process_result: "'Hello' in r'''$(result)''' and 'PASS' in r'''$(result)''' "}
- console:
name: Console closure
execute_on_stop: true
console_name: term
key: $(test)_PASS
steps:
- close: term

12
test/validation/items/cycle/cycle.py Normal file
View File

@@ -0,0 +1,12 @@
def donothing():
return 0
def checkloopparam(loop_param):
if not(loop_param==12 or loop_param==20 or loop_param==30):
raise
return 0
def exitcondition(loop):
if loop == 3:
return True

1
test/validation/items/cycle/param.yaml Normal file
View File

@@ -0,0 +1 @@
no_param: Null

34
test/validation/items/cycle/test.tum Normal file
View File

@@ -0,0 +1,34 @@
- loop:
name: Cycle number of loops
iterator: 10
key: $(test)_PASS
steps:
- py_func:
name: do nothing
file: $(test_path)$(psep)cycle.py
func_name: donothing
- loop:
name: Cycle iterating on list
iterator: [12, 20, 30]
key: $(test)_PASS
steps:
- py_func:
name: check loop param
file: $(test_path)$(psep)cycle.py
func_name: checkloopparam
param:
- $(loop_param)
- loop:
name: Infinite loop with exit condition
key: $(test)_PASS
steps:
- py_func:
name: do nothing
file: $(test_path)$(psep)cycle.py
func_name: donothing
exit_condition:
file: $(test_path)$(psep)cycle.py
func_name: exitcondition

BIN
test/validation/items/dialogs/IMG_20140213_171455.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 969 KiB

1
test/validation/items/dialogs/param.yaml Normal file
View File

@@ -0,0 +1 @@
no_param: Null

61
test/validation/items/dialogs/test.tum Normal file
View File

@@ -0,0 +1,61 @@
- dialog_image:
name: dialog image PASS
condition: $(validation_dialogs)
question: click ok if you see the image
key: $(test)_PASS
filename: $(test_path)$(psep)IMG_20140213_171455.jpg
- dialog_image:
name: dialog image FAIL
condition: $(validation_dialogs)
question: click cancel
key: $(test)_FAIL
filename: $(test_path)$(psep)IMG_20140213_171455.jpg
- dialog_references:
name: dialog_reference PASS
condition: $(validation_dialogs)
key: $(test)_PASS
question: click ok
- dialog_references:
name: dialog_reference FAIL
condition: $(validation_dialogs)
key: $(test)_FAIL
question: click cancel
- dialog_value:
name: dialog_value PASS
condition: $(validation_dialogs)
key: $(test)_PASS
question: enter 123 and click ok
- dialog_value:
name: dialog_value empty FAIL
condition: $(validation_dialogs)
key: $(test)_FAIL
question: enter nothing and click ok
- dialog_value:
name: dialog_value canceled FAIL
condition: $(validation_dialogs)
key: $(test)_FAIL
question: enter nothing and click cancel
- dialog_message:
name: dialog_message PASS
condition: $(validation_dialogs)
key: $(test)_PASS
question: click ok
- dialog_question:
name: dialog_question PASS
condition: $(validation_dialogs)
key: $(test)_PASS
question: click yes
- dialog_question:
name: dialog_question FAIL
condition: $(validation_dialogs)
key: $(test)_FAIL
question: click no

3
test/validation/items/expanse/expanse.py Normal file
View File

@@ -0,0 +1,3 @@
def echo(param):
return param

12
test/validation/items/expanse/param.yaml Normal file
View File

@@ -0,0 +1,12 @@
expanse_select: blo
expanse_table_bla: ["abcd", 1234, [5, 6, 7, 8]]
expanse_index_bla: 0
expanse_table_blo: ["efgh", 9012, [3, 4, 5, 6]]
expanse_index_blo: 1
expanse_index: $(expanse_index_$(expanse_select))
expanse_table: $(expanse_table_$(expanse_select))
expanse_eval: <@$(expanse_index) == 1@>

64
test/validation/items/expanse/test.tum Normal file
View File

@@ -0,0 +1,64 @@
- check:
name: Check variables expansion is correct (PASS)
key: $(test)_PASS
values:
- $(expanse_index) == 1
- $(expanse_table)[$(expanse_index)] == 9012
- $(expanse_eval) == True
- let:
name: Dynamic variables expansion
key: $(test)_PASS
values:
- expanse_select: <@"$(expanse_select)".replace("o", "a")@>
- expanse_index: $(expanse_index_$(expanse_select))
- expanse_table: $(expanse_table_$(expanse_select))
- expanse_eval: <@$(expanse_index) == 1@>
- check:
name: Check variables expansion is correct (PASS)
key: $(test)_PASS
values:
- $(expanse_index) == 0
- $(expanse_table)[$(expanse_index)] == "abcd"
- $(expanse_eval) == False
- let:
name: Complex variables expansion
key: $(test)_PASS
values:
- var1: expanse
- var2: var
- var3: bla
- var4: blo
- expanse_var_bla: 3
- expanse_blo_var: 5
- expanse_complex: <@<@$(expanse_$(var2)_$(var3))*6@> + <@4*$($(var1)_$(var4)_$(var2))@>@>
- check:
name: Check complex variables expansion is correct (PASS)
key: $(test)_PASS
values:
- $(expanse_complex) == 38
- let:
name: Variables expansion in object
key: $(test)_PASS
values:
- expanse_key: b
- expanse_var: 3
- expanse_var_2: 6
- expanse_object:
[
{ a: $(expanse_var_2), $(expanse_key): <@2**3@> },
{
'<@"bla".replace("a", "o")@>':
[<@$(expanse_var)*$(expanse_var_2)@>, 25],
},
]
- check:
name: Check complex variables expansion is correct (PASS)
key: $(test)_PASS
values:
- '$(expanse_object) == [{"a": 6, "b": 8}, {"blo": [18, 25]}]'

26
test/validation/items/func/func.py Normal file
View File

@@ -0,0 +1,26 @@
import py_func.tm as tm
class ValidationTest(tm.FunctionItem):
def exec(self, param):
print(str(param))
return 0
def donothing():
return 0
def assertparam(param):
assert param
return 0
def checkglobal(param):
assert param=='test parameter'
return 0
def should_not_be_called(param):
raise
def echo(param):
return param
def tuple_return(first, second):
return first, second

1
test/validation/items/func/param.yaml Normal file
View File

@@ -0,0 +1 @@
skipped_test_item: ['skipped_checkglobal']

183
test/validation/items/func/test.tum Normal file
View File

@@ -0,0 +1,183 @@
- let:
name: func test constants,
values:
test parameter: test parameter
- py_func:
name: pass func
key: $(test)_PASS
file: $(test_path)$(psep)func.py
func_name: assertparam
param:
- true
- py_func:
name: fail func
key: $(test)_FAIL
file: $(test_path)$(psep)func.py
func_name: assertparam
param:
- false
- py_func:
name: fail func with expected result "FAIL"
key: $(test)_PASS
file: $(test_path)$(psep)func.py
func_name: assertparam
param:
- false
expected_result: FAIL
- py_func:
name: pass func with expected result FAIL
key: $(test)_FAIL
file: $(test_path)$(psep)func.py
func_name: assertparam
param:
- true
expected_result: FAIL
- py_func:
name: expected -1
key: $(test)_PASS
file: $(test_path)$(psep)func.py
func_name: echo
param:
- -1
expected_result: -1
- py_func:
name: expected eval
key: $(test)_PASS
file: $(test_path)$(psep)func.py
func_name: echo
param:
- -1
expected_result: "354848436 - 354848437"
- py_func:
name: expected table
key: $(test)_PASS
file: $(test_path)$(psep)func.py
func_name: echo
param:
- [-1, a, {toto: tata}]
expected_result: "[-1, 'a', {'toto': 'tata'}]"
- py_func:
name: global param func
key: $(test)_PASS
file: $(test_path)$(psep)func.py
func_name: checkglobal
param:
- $(test parameter)
- let:
name: python2func
key: $(test)_PASS
values:
- py: $(test_path)$(psep)func.py
- py_func:
name: global param func
key: $(test)_PASS
file: $(py)
func_name: checkglobal
param:
- $(test parameter)
- py_func:
name: skipped_checkglobal
file: $(test_path)$(psep)func.py
func_name: should_not_be_called
param:
- $(test parameter)
- py_func:
name: skipped true
file: $(test_path)$(psep)func.py
func_name: checkglobal
skipped: true
param:
- $(test parameter)
- py_func:
name: skipped true
file: $(test_path)$(psep)func.py
func_name: checkglobal
skipped: true
param:
- $(test parameter)
- py_func:
name: skipped 1
file: $(test_path)$(psep)func.py
func_name: checkglobal
skipped: 1
param:
- $(test parameter)
- py_func:
name: FunctionItem test
file: $(test_path)$(psep)func.py
func_name: ValidationTest
param:
- $(test parameter)
- group:
name: Function results check
steps:
- group:
name: Function result failure
steps:
- py_func:
name: int failure
key: $(test)_FAIL
file: $(test_path)$(psep)func.py
func_name: echo
param: [-1]
- py_func:
name: float failure
key: $(test)_FAIL
file: $(test_path)$(psep)func.py
func_name: echo
param: [-1.3]
- py_func:
name: String failure
key: $(test)_FAIL
file: $(test_path)$(psep)func.py
func_name: echo
param: [ "FAIL" ]
- py_func:
name: Tuple int,str failure
key: $(test)_FAIL
file: $(test_path)$(psep)func.py
func_name: tuple_return
param: [ -1, "Got a failure" ]
- group:
name: Functions result success
steps:
- py_func:
name: int success
key: $(test)_PASS
file: $(test_path)$(psep)func.py
func_name: echo
param: [0]
- py_func:
name: float success
key: $(test)_PASS
file: $(test_path)$(psep)func.py
func_name: echo
param: [0.3]
- py_func:
name: String success
key: $(test)_PASS
file: $(test_path)$(psep)func.py
func_name: echo
param: [ "Something that is not only strictly FAIL" ]
- py_func:
name: Tuple int,str success
key: $(test)_PASS
file: $(test_path)$(psep)func.py
func_name: tuple_return
param: [ 0, "OK" ]

1
test/validation/items/git/param.yaml Normal file
View File

@@ -0,0 +1 @@
no_param: Null

11
test/validation/items/git/test.tum Normal file
View File

@@ -0,0 +1,11 @@
- git:
name: Testium repo
key: $(test)_PASS
repo: $(test_directory)
- git:
name: Testium repo
key: $(test)_PASS
repo:
- $(test_directory)
- $(test_directory)

8
test/validation/items/include/inc no template/my first include.tum Normal file
View File

@@ -0,0 +1,8 @@
- py_func:
name: My first include test
file: $(test_path)$(psep)include.py
func_name: ValidationTest
key: $(test)_PASS
param:
- $(test parameter)

7
test/validation/items/include/inc with template/my second include.tum Normal file
View File

@@ -0,0 +1,7 @@
- py_func:
name: {{ func_title }}
file: $(test_path)$(psep)include.py
func_name: ValidationTest
key: $(test)_PASS
param:
- {{ func_param }}

7
test/validation/items/include/inc with template/my_3d_include.tum Normal file
View File

@@ -0,0 +1,7 @@
- let: {name: Test param inclusion 1, values: [inclusion: $({{ dd }}_inc)]}
# note the Jinja comments : {# #}
# The following construction is not allowed and will fail to load:
# {#
# - let: {name: Test param inclusion 2, values: [inclusion: {{ $(inc)_inc }}]}
# #}

7
test/validation/items/include/include.py Normal file
View File

@@ -0,0 +1,7 @@
import py_func.tm as tm
class ValidationTest(tm.FunctionItem):
def exec(self, param):
print(str(param))
return 0

3
test/validation/items/include/param.yaml Normal file
View File

@@ -0,0 +1,3 @@
# inc_no template: "inc no template" # Both commented parma cannot be handled as this configuration file is loaded dynamically
# inc_with_template: "inc with template" # through the template mechanism
test_parameter: My first include test parameter

12
test/validation/items/include/test.tum Normal file
View File

@@ -0,0 +1,12 @@
- !include inc no template/my first include.tum
- !include $(inc_no_template)/my first include.tum
- !include {file: inc with template/my second include.tum, func_title: My second include test, func_param: My second include test parameter}
- !include
file: $(inc_with_template)/my second include.tum
func_title: My second include test
func_param: My second include test parameter
- let: {name: Declare param for inclusion, values: [inc: Dali, Dali_inc: Dalida]}
- !include {file: $(inc_with_template)/my_3d_include.tum, dd: $(inc)}

15
test/validation/items/jsonrpc/jrpces.ini Normal file
View File

@@ -0,0 +1,15 @@
[general]
default=default
[jsonrpc_tcp]
host=0.0.0.0
port=4321
[jsonrpc_html]
host=0.0.0.0
port=4322
[jsonrpc_udp]
host=0.0.0.0
port=4323

1
test/validation/items/jsonrpc/param.yaml Normal file
View File

@@ -0,0 +1 @@
skip_tcp: True

333
test/validation/items/jsonrpc/test.tum Normal file
View File

@@ -0,0 +1,333 @@
- console:
name: json rpc echo server
doc: check if the jsonrpc echo server is installed
console_name: jrpces
key: $(test)_PASS
steps:
- open:
protocol: terminal
- read_until: {expected: $(terminal_prompt), timeout: 1, no_fail: True}
- writeln: which jrpces
- read_until: {expected: jrpces, timeout: 2}
- group:
name: jsonrpc tests
condition: "'/jrpces' in r'''$(cn_json rpc echo server)'''"
steps:
- console:
name: Start the json rpc echo server
console_name: jrpces
key: $(test)_PASS
steps:
- writeln: jrpces -c {{include_directory}}/jrpces.ini
- read_until: {expected: $(terminal_prompt), timeout: 1, no_fail: True}
- console:
name: Open the raw tcp Console
skipped: $(skip_tcp)
console_name: jsonrpc_server
doc: Opening the RAW TCP console
key: $(test)_PASS
steps:
- open:
protocol: rawtcp
tcp_host: localhost
tcp_port: 4321
- json_rpc:
name: JSONRPC console Query waiting for reception
skipped: $(skip_tcp)
key: $(test)_PASS
console: {name : jsonrpc_server}
timeout: 1
steps:
- query:
method: echo
params:
- Hello World
- {a: 1, b: "hello"}
- json_rpc:
name: JSONRPC console Query not waiting (only send)
skipped: $(skip_tcp)
key: $(test)_PASS
console: {name : jsonrpc_server}
timeout: 1
steps:
- query:
method: echo
params:
- {b: "olleh", a: -1}
- World Hello
id: 3095372
no_wait: True
- sleep: {name: "Small delay for the test", timeout: 1, skipped: $(skip_tcp)}
- json_rpc:
name: JSONRPC console Reception
skipped: $(skip_tcp)
key: $(test)_PASS
console: {name : jsonrpc_server}
timeout: 1
steps:
- receive:
id: 3095372
- console:
name: Close the raw tcp console
skipped: $(skip_tcp)
console_name: jsonrpc_server
doc: Opening the RAW TCP console
key: $(test)_PASS
execute_on_stop: True
steps:
- close:
- json_rpc:
name: JSONRPC UDP query waiting for reception
key: $(test)_PASS
udp: {server: localhost, snd_port: 4323, rcv_port: 8765}
timeout: 1
steps:
- open:
- query:
name: echo
method: echo
params:
- Hello World
- {a: 1, b: "hello"}
timeout: 1
- close:
- json_rpc:
name: Failing JSONRPC UDP query waiting for reception (returning an error)
key: $(test)_FAIL
udp: {server: localhost, snd_port: 4323, rcv_port: 8765}
timeout: 1
steps:
- open:
- query:
method: echo2
params:
- Hello World
- {a: 1, b: "hello"}
timeout: 1
- close:
- json_rpc:
name: JSONRPC UDP query waiting for reception of an expected error
key: $(test)_PASS
udp: {server: localhost, snd_port: 4323, rcv_port: 8765}
timeout: 1
steps:
- open:
- query:
method: echo2
params:
- Hello World
- {a: 1, b: "hello"}
expected_result: {code: -32000, message: function not found}
- close:
- json_rpc:
name: Failing UDP JSONRPC query timeout elapses (wrong udp port)
doc: |
Failing JSONRPC UDP query waiting for reception and checking result
and timeout elapses (wrong udp port)
key: $(test)_FAIL
udp: {server: localhost, snd_port: 4326, rcv_port: 48393}
timeout: 1
steps:
- open:
- query:
method: echo
params:
- Hello World
- {a: 1, b: hello}
timeout: 0.5
- close:
- json_rpc:
name: JSONRPC UDP query not waiting (only send)
key: $(test)_PASS
udp: {server: localhost, snd_port: 4323, rcv_port: 8765}
timeout: 1
steps:
- open:
- query:
method: echo
params:
- {b: "olleh", a: -1}
- World Hello
id: 3095372
no_wait: True
- sleep: {name: "Small delay for the test", timeout: 1}
- json_rpc:
name: JSONRPC UDP Reception
key: $(test)_PASS
udp: {server: localhost, snd_port: 4323, rcv_port: 8765}
timeout: 1
steps:
- receive:
id: 3095372
- close:
- json_rpc:
name: JSONRPC UDP query not waiting (only send)
key: $(test)_PASS
udp: {server: localhost, snd_port: 4323, rcv_port: 8765}
timeout: 1
steps:
- open:
- query:
method: echo2
params:
- {b: "olleh", a: -1}
- World Hello
id: 3095372
no_wait: True
- sleep: {name: "Small delay for the test", timeout: 1}
- json_rpc:
name: Failing JSONRPC UDP Reception (returning an error)
key: $(test)_FAIL
udp: {server: localhost, snd_port: 4323, rcv_port: 8765}
timeout: 1
steps:
- receive:
id: 3095372
timeout: 1
- close:
- json_rpc:
name: UDP JSONRPC query waiting and checking
doc: JSONRPC UDP query waiting for reception and checking result
key: $(test)_PASS
udp: {server: localhost, snd_port: 4323, rcv_port: 48393}
timeout: 1
steps:
- open:
- query:
method: echo
params:
- Hello World
- {a: 1, b: hello}
expected_result: [[Hello World, {a: 1, b: hello}], {}]
timeout: 1
- close:
- json_rpc:
name: Failing UDP JSONRPC query waiting and checking
doc: JSONRPC UDP query waiting for reception and checking result
key: $(test)_FAIL
udp: {server: localhost, snd_port: 4323, rcv_port: 48393}
timeout: 1
steps:
- open:
- query:
method: echo
params:
- Hello World
- {a: 1, b: hello}
expected_result: [[], {}]
timeout: 1
- close:
- json_rpc:
name: UDP JSONRPC query not waiting (for checking)
doc: JSONRPC UDP query not waiting, with the purpose to check the result at reception
key: $(test)_PASS
udp: {server: localhost, snd_port: 4323, rcv_port: 9876}
timeout: 1
steps:
- open:
- query:
method: echo
params:
- {b: "olleh", a: -1}
- World Hello
id: 3095372
no_wait: True
- sleep: {name: "Small delay for the test", timeout: 1}
- json_rpc:
name: UDP JSONRPC reception checking
doc: JSONRPC UDP Reception and checking result
key: $(test)_PASS
udp: {server: localhost, snd_port: 4323, rcv_port: 9876}
timeout: 1
steps:
- receive:
id: 3095372
timeout: 1
expected_result: [[{b: olleh, a: -1}, World Hello], {}]
- close:
- json_rpc:
name: Failing UDP JSONRPC reception checking
doc: JSONRPC UDP Reception and checking result
key: $(test)_FAIL
udp: {server: localhost, snd_port: 4323, rcv_port: 9876}
timeout: 1
steps:
- receive:
id: 3095372
timeout: 1
expected_result: [[{b: ollhe, a: -1}, World Hello], {}]
- close:
- json_rpc:
name: UDP JSONRPC query waiting and evaluating result
doc: |
JSONRPC UDP query waiting for reception and checking result with
replacing $(result) and evaluating string.
key: $(test)_PASS
udp: {server: localhost, snd_port: 4323, rcv_port: 48393}
timeout: 1
steps:
- open:
- query:
method: echo
params:
- Hello World
- {a: 1, b: hello}
process_result: $(result)[0][1]['a']
expected_result: 1
timeout: 1
- close:
- json_rpc:
name: Failing UDP JSONRPC query waiting and evaluating result
doc: |
JSONRPC UDP query waiting for reception and checking result with
replacing $(result) and evaluating string.
key: $(test)_FAIL
udp: {server: localhost, snd_port: 4323, rcv_port: 48393}
timeout: 1
steps:
- open:
- query:
method: echo
params:
- Hello World
- {a: 1, b: hello}
expected_result: "$(result)[0][1]['a'] == 0"
timeout: 1
- close:
- console:
name: Stop json rpc echo server
doc: check if the jsonrpc echo server is installed
console_name: jrpces
execute_on_stop: True
key: $(test)_PASS
steps:
- close:
protocol: terminal

19
test/validation/items/let/let.py Normal file
View File

@@ -0,0 +1,19 @@
import py_func.tm as tm
def donothing():
return 0
def checkloopparam(loop_param):
if not (loop_param == 12 or loop_param == 20
or loop_param == 30):
raise
return 0
def checkGlobalDic(param, expect):
if tm.gd(param) != expect:
raise Exception("Expected {} for {} but got {}".
format(expect, param, tm.gd(param)))
return 0

3
test/validation/items/let/param.yaml Normal file
View File

@@ -0,0 +1,3 @@
no_param: Null
overwrite_me: True
test_overwrite_me: False

95
test/validation/items/let/test.tum Normal file
View File

@@ -0,0 +1,95 @@
- loop:
name: Cycle number of loops
iterator: 10
key: $(test)_PASS
steps:
- py_func:
name: do nothing
file: $(test_path)$(psep)let.py
func_name: donothing
- let:
name: Let it be
values:
it: $(loop_param)
eval:
- be: "$(loop_param) == $(it)"
- loop:
name: Cycle iterating on list
iterator: [12, 20, 30]
key: $(test)_PASS
steps:
- py_func:
name: check loop param
file: $(test_path)$(psep)let.py
func_name: checkloopparam
param:
- $(loop_param)
- let:
name: Let it be
values:
- it: $(loop_param)
eval:
- be: "$(loop_param) == $(it)"
- let:
name: Get time
key: $(test)_PASS
values:
- loop_t0: $(ts_start_Cycle iterating on list)
- loop_t1: $(ts_end_Cycle iterating on list)
- loop_duration: $(ts_duration_Cycle iterating on list)
- let:
name: Get parameter file value
key: $(test)_PASS
eval:
- test_overwrite_me: "$(overwrite_me) == True"
- py_func:
name: Check global dic pass
file: $(test_path)$(psep)let.py
func_name: checkGlobalDic
param:
- test_overwrite_me
- True
- let:
name: Overwrite parameter file value
key: $(test)_PASS
values:
- overwrite_me: False
- py_func:
name: Check global dic fail
file: $(test_path)$(psep)let.py
func_name: checkGlobalDic
key: $(test)_FAIL
param:
- overwrite_me
- True
expected_result: "$(overwrite_me) == False"
- py_func:
name: Check global dic fail
file: $(test_path)$(psep)let.py
func_name: checkGlobalDic
key: $(test)_PASS
param:
- overwrite_me
- True
expected_result: fail
- let:
name: Evaluate Overwriting parameter file value
key: $(test)_PASS
eval:
- test_overwrite_me: '"$(overwrite_me)" == True'
- check:
name: Check Overwriting parameter file value
key: $(test)_PASS
values:
- $(test_overwrite_me) == False

1
test/validation/items/plot/param.yaml Normal file
View File

@@ -0,0 +1 @@
no_param: Null

13
test/validation/items/plot/plot.py Normal file
View File

@@ -0,0 +1,13 @@
import random
import py_func.tm as tm
from py_func.tm import FunctionItem
def random_value():
return random.random()
class LastValues(FunctionItem):
def exec(self, name):
res = tm.last_plot_value(name)
self.reportValue('my_reported_value', res)
print("Last plot value: {}".format(res))
return res

74
test/validation/items/plot/test.tum Normal file
View File

@@ -0,0 +1,74 @@
- plot:
name: Open the plot
condition: $(validation_dialogs)
key: $(test)_PASS
plot_name: Mon Plot
steps:
- open:
log_path: $(validation_report_path)
- plot:
name: Add periodic to the plot
condition: $(validation_dialogs)
key: $(test)_PASS
plot_name: Mon Plot
steps:
- periodic:
period: 1
file: $(test_path)$(psep)plot.py
func_name: random_value
eval: '{"periodic": $(result)}'
- sleep:
name: sleep
condition: $(validation_dialogs)
dialog: true
timeout: 3
- loop:
name: Add of other data in the plot
condition: $(validation_dialogs)
iterator: 10
steps:
- plot:
name: Add to the plot
key: $(test)_PASS
plot_name: Mon Plot
steps:
- add:
value1: $(loop_index)
value2: $(loop_index)+2
- sleep:
name: sleep between values
timeout: 1
- py_func:
name: last plot values
key: $(test)_PASS
file: $(test_path)$(psep)plot.py
func_name: LastValues
param:
- Mon Plot
- plot:
name: Export
execute_on_stop: True
condition: $(validation_dialogs)
key: $(test)_PASS
plot_name: Mon Plot
steps:
- export: $(validation_report_path)/plot_export.pdf
- export: $(validation_report_path)/plot_export.csv
- plot:
name: Close the plot
execute_on_stop: True
condition: $(validation_dialogs)
key: $(test)_PASS
plot_name: Mon Plot
steps:
- close:
wait_dialog_exit: True
timeout: 60

25
test/validation/items/report.tum Normal file
View File

@@ -0,0 +1,25 @@
- report:
name: Expected PASS $(test) test
export:
- text:
path: $(validation_report_path)$(psep)$(test)_PASS.txt
key: $(test)_PASS
- html:
path: $(validation_report_path)$(psep)$(test)_PASS.html
key: $(test)_PASS
- junit:
path: $(validation_report_path)$(psep)$(test)_PASS.junit
key: $(test)_PASS
- report:
name: Expected FAIL $(test) test
export:
- text:
path: $(validation_report_path)$(psep)$(test)_FAIL.txt
key: $(test)_FAIL
- html:
path: $(validation_report_path)$(psep)$(test)_FAIL.html
key: $(test)_FAIL
- junit:
path: $(validation_report_path)$(psep)$(test)_FAIL.junit
key: $(test)_FAIL

1
test/validation/items/sleep/param.yaml Normal file
View File

@@ -0,0 +1 @@
no_param: Null

17
test/validation/items/sleep/test.tum Normal file
View File

@@ -0,0 +1,17 @@
- sleep:
name: Sleep timeout with dialogs
condition: $(validation_dialogs)
key: $(test)_PASS
timeout: 3
dialog: true
- sleep:
name: Sleep timeout without dialog
key: $(test)_PASS
timeout: 3.0
- sleep:
name: Sleep timeout in textual format
skipped: true
key: $(test)_PASS
timeout: 1h 3m 2s

1
test/validation/items/unittest/param.yaml Normal file
View File

@@ -0,0 +1 @@
no_param: Null

13
test/validation/items/unittest/test.tum Normal file
View File

@@ -0,0 +1,13 @@
- unittest_file:
name: Unittest item
test_file: {{include_directory}}/unittest.py
key: $(test)_PASS
test_method: test_01_pass
- unittest_file:
name: Unittest item
test_file: {{include_directory}}/unittest.py
key: $(test)_FAIL
test_method:
- test_04_disabled
- test_03_fail

37
test/validation/items/unittest/unittest.py Normal file
View File

@@ -0,0 +1,37 @@
import unittest
from time import sleep
class MyUnitTests(unittest.TestCase):
@unittest.skip("test skipped")
def test_00_skipped(self):
''' Test 00 is skipped
'''
sleep(0.5)
def test_01_pass(self):
''' Test 01 is passed and adds a report key
'''
self.reported_values['key reported']= 'value_reported'
sleep(0.5)
def test_02_pass(self):
''' Test 02 is passed and adds a report key
'''
self.reported_values['key reported']= 'toto'
sleep(0.5)
def test_03_fail(self):
''' Test 03 is fail by unittest method
'''
sleep(0.5)
self.fail(msg='Fail message')
def test_04_disabled(self):
''' Test 04 is disabled
'''
sleep(0.5)
def test_05_crash(self):
''' Test 05 crashes
'''
print(crash)

51
test/validation/main.tum Normal file
View File

@@ -0,0 +1,51 @@
config_file:
- param.yaml
{% for item in items %}
- "items/{{ item }}/param.yaml"
{% endfor %}
main:
name: Testium validation suite
steps:
- group:
name: Test preparation
steps:
- let:
name: Set test variables for Linux
condition: "'$(os)' == 'Linux'"
values:
- terminal_prompt: $(linux_prompt)
- psep: "/"
- let:
name: Set test variables for Windows
condition: "'$(os)' == 'Windows'"
values:
- terminal_prompt: $(windows_prompt)
- psep: "\\"
- group:
name: Group of tests
steps:
{% for item in items %}
# item test
- let: {name: {{ item }} test constants, values: {test: {{ item }}, test_path: items/$(test)}}
- group:
name: {{ item }} test
steps:
- !include items/{{ item }}/test.tum
- !include items/report.tum
{% endfor %}
report:
enabled: True
log_stored: True
export:
junit:
path: $(validation_report_path)
file_name: $(validation_report_file).junit
sqlite:
file_name: $(validation_report_file).sqlite
path: $(validation_report_path)

31
test/validation/param.yaml Normal file
View File

@@ -0,0 +1,31 @@
# All sub directory in items are evaluated as a list
items: <@ [os.path.basename(f.path) for f in os.scandir(os.path.join(r"$(test_directory)", "items")) if f.is_dir()] @>
# - common
# - check
# - console
# - cycle
# - dialogs
# - expanse
# - func
# - git
# - plot
# - include
# - jsonrpc
# - let
# - sleep
# - unittest
# Parameters of the validation execution
validation_report_path: $(test_directory)/../tmp/
validation_report_file: validation
# parameter to enable validation of plotical items
validation_dialogs: true
# various parameters
windows_prompt: ">"
linux_prompt: "$ "
#Added param to show handling of the include with statically loaded parameter
inc_no_template: "inc no template"
inc_with_template: "inc with template"

109
test/validation/post_execution.py Normal file
View File

@@ -0,0 +1,109 @@
import os
import py_func.tm as tm
import textwrap
import sqlite3
from junit_xml import TestSuite, TestCase
from interpreter.test_items.test_result import TestValue
def _prepare_file_to_save(file_name, file_ext=""):
iname = file_name
if file_ext != "":
iname = os.path.splitext(file_name)[0] + file_ext
if os.path.isfile(iname):
i = 0
fname = iname
while os.path.isfile(fname):
i += 1
fname = iname + "-" + str(i) + ".saved"
os.rename(iname, fname)
return iname
def _get_testSuite(test, cur):
test_cases = []
failures = 0
failed_results = cur.execute(
f"SELECT test_name, result, report_key, duration, log, data FROM tests WHERE report_key LIKE '{test}_%'"
).fetchall()
for result in failed_results:
tc = TestCase(
name=f"[{test}] {result[0]}",
elapsed_sec=result[3],
stdout=result[4],
log=result[5],
)
# Check the results of all
if result[1] == str(TestValue.NORUN):
tc.add_skipped_info("The test has not being runned")
elif result[2] == f"{test}_PASS":
if result[1] == str(TestValue.FAILURE):
failures += 1
print(f"Item [{test}] Failing on '{result[0]}' : The test should PASS")
print("*" * 80)
print(textwrap.indent(result[4], prefix="* "))
print("*" * 80)
tc.add_error_info("The test should PASS!\n\n" + result[4])
elif result[2] == f"{test}_FAIL":
if result[1] == str(TestValue.SUCCESS):
failures += 1
print(
f"Item [{test}] Failing on '{result[0]}' : \n\tThe test should FAIL"
)
print("*" * 80)
print(textwrap.indent(result[4], prefix="* "))
print("*" * 80)
tc.add_error_info("The test should FAIL!\n\n" + result[4])
# add to the test cases
test_cases.append(tc)
return failures, TestSuite(test, test_cases)
def exec():
print("\n")
print("*" * 80)
print("* Post execution started")
print("*" * 80)
# Get the info
report = (
str(tm.gd("validation_report_path"))
+ str(tm.gd("validation_report_file"))
+ ".sqlite"
)
enabled_tests = tm.gd("items")
# Open the database
con = sqlite3.connect(report)
cur = con.cursor()
# Get the testsuit for every parts
failures = 0
for test in enabled_tests:
failed, ts = _get_testSuite(test, cur)
failures += failed
# write to the file
junit_report = report.replace(".sqlite", f"-{test}.xml")
print(junit_report)
_prepare_file_to_save(junit_report)
with open(junit_report, "w") as f:
f.write(TestSuite.to_xml_string([ts]))
# cleanup
con.close()
print("*" * 80)
if failures == 0:
print("* Post execution finished : SUCCESS")
else:
print(f"* Post execution finished: {failures} test FAILED")
print("*" * 80)
def post_exec():
exec()
def post_exec_fail():
exec()