diff --git a/schema/test_schema/check.tum b/schema/test_schema/check.tum new file mode 100644 index 0000000..9f45bf3 --- /dev/null +++ b/schema/test_schema/check.tum @@ -0,0 +1,96 @@ +config_file: +- param.yaml +- items/check/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: check test constants + values: + - test: check + - test_path: items/$(test) + - group: + name: check test + steps: + - sequence: + steps: + - py_func: + file: $(test_path)$(psep)check.py + func_name: echo + key: $(test)_PASS + name: Dummy_int + param: + - 2 + - py_func: + file: $(test_path)$(psep)check.py + func_name: echo + key: $(test)_PASS + name: Dummy_str + param: + - my taylor is rich + - check: + key: $(test)_PASS + name: Check condition on existing variable (PASS) + values: + - <| $(pfn_Dummy_int) > 1 |> + - check: + key: $(test)_FAIL + name: Check condition on existing variable (FAIL) + values: + - <| "tailor" in "$(pfn_Dummy_str)" |> + filename: /home/renish/workspace/testium/code/test/validation/items/check/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/common.tum b/schema/test_schema/common.tum new file mode 100644 index 0000000..25f5d2e --- /dev/null +++ b/schema/test_schema/common.tum @@ -0,0 +1,453 @@ +config_file: +- param.yaml +- items/common/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: common test constants + values: + test: common + test_path: items/$(test) + - group: + name: common test + steps: + - sequence: + steps: + - group: + name: Results + steps: + - sequence: + steps: + - group: + name: Expected Result + steps: + - py_func: + expected_result: true + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_PASS + name: Return True expect True + param: + - true + - py_func: + expected_result: false + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_FAIL + name: Return True expect False (must + fail) + param: + - true + - py_func: + expected_result: None + file: $(test_path)$(psep)results$(psep)results.py + func_name: return_none + key: $(test)_PASS + name: Return None expect None + - py_func: + expected_result: PASS + file: $(test_path)$(psep)results$(psep)results.py + func_name: return_none + key: $(test)_PASS + name: Return None expect PASS + - py_func: + expected_result: 14 + file: $(test_path)$(psep)results$(psep)results.py + func_name: return_none + key: $(test)_FAIL + name: Return None expect 14 (must fail) + - group: + name: Expected Result Last test result + steps: + - py_func: + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_PASS + name: result is 28 + param: + - 28 + - py_func: + expected_result: $(last_step_result) + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_PASS + name: check that the last test result + is 28 + param: + - 28 + - group: + name: Expected result Failure raised issue + steps: + - py_func: + file: $(test_path)$(psep)results$(psep)results.py + func_name: raise_issue + key: $(test)_FAIL + name: Raise an issue (must fail) + param: + - $(str_example) + - py_func: + expected_result: FAIL + file: $(test_path)$(psep)results$(psep)results.py + func_name: raise_issue + key: $(test)_PASS + name: Raise an issue and expected the + test to be FAIL + param: + - $(str_example) + - py_func: + expected_result: FAIL + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_FAIL + name: Return a String expect a FAILURE + (must fail) + param: + - $(str_example) + - group: + name: process result + steps: + - py_func: + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_PASS + name: Process result equal String + param: + - $(str_example) + process_result: '''$(str_example)'' == + ''$(result)''' + - py_func: + expected_result: true + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_FAIL + name: Process result string in the result + (must fail) + param: + - $(str_example) + process_result: '''44'' in ''$(result)''' + - py_func: + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_PASS + name: Save the result in a global variable + param: + - 44 + store_result: process_result_value + - py_func: + expected_result: $(process_result_value) + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_PASS + name: Check the saved global variable + param: + - 44 + - py_func: + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_PASS + name: store_result with process_result + param: + - $(str_example) + process_result: '''$(result)''.upper()' + store_result: upper_str_example + - py_func: + expected_result: $(upper_str_example) + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_PASS + name: Check store_result with process_result + param: + - $(str_example) + process_result: '''$(result)''.upper()' + - let: + key: $(test)_PASS + name: "store_result on let item (None\ + \ value \u2192 stores PASS)" + store_result: let_store_result + values: + - dummy: 0 + - py_func: + expected_result: $(let_store_result) + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_PASS + name: Check store_result on let stores + PASS + param: + - PASS + - py_func: + expected_result: FAIL + file: $(test_path)$(psep)results$(psep)results.py + func_name: return_none + key: $(test)_FAIL + name: "store_result on failing test (None\ + \ value \u2192 stores FAIL)" + store_result: none_fail_store_result + - py_func: + expected_result: $(none_fail_store_result) + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_PASS + name: Check store_result on failing test + stores FAIL + param: + - FAIL + - py_func: + expected_result: FAIL + file: $(test_path)$(psep)results$(psep)results.py + func_name: return_none + key: $(test)_PASS + name: "store_result with no_fail (None\ + \ value \u2192 stores real FAIL, not\ + \ forced PASS)" + no_fail: true + store_result: none_nofail_store_result + - py_func: + expected_result: $(none_nofail_store_result) + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_PASS + name: Check store_result with no_fail + stores real FAIL + param: + - FAIL + - py_func: + file: $(test_path)$(psep)results$(psep)results.py + func_name: return_none + key: $(test)_FAIL + name: Process result when result is None + (must fail) + process_result: $(result) is None + - group: + name: no_fail result + steps: + - py_func: + expected_result: false + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_PASS + name: Return True expect False but no_fail=True + no_fail: true + param: + - true + - py_func: + expected_result: false + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_FAIL + name: Return True expect False but no_fail=False + (must fail) + no_fail: false + param: + - true + - py_func: + expected_result: false + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_PASS + name: Return True expect False but no_fail + expansed + no_fail: <| bool(0) == False |> + param: + - true + - py_func: + expected_result: false + file: $(test_path)$(psep)results$(psep)results.py + func_name: echo + key: $(test)_FAIL + name: Return True expect False but no_fail + expansed (must fail) + no_fail: <| bool(1) == False |> + param: + - true + filename: /home/renish/workspace/testium/code/test/validation/items/common/results/test.tum + - group: + name: Conditional + steps: + - sequence: + steps: + - loop: + doc: This loop illustrate the way to exit on + a condition. + exit_condition: + value: <| $(pfn_Echo function) > 3 |> + key: $(test)_PASS + name: Infine loop with conditional exit + steps: + - sleep: + name: small wait + timeout: 0.2 + - py_func: + file: $(test_path)$(psep)conditional$(psep)conditional.py + func_name: echo + key: $(test)_PASS + name: Echo function + param: + - $(loop_param) + stop_on_failure: false + - let: + key: $(test)_PASS + name: let + values: + - conditional_exec: <| random.randint(1, 2) |> + - console: + condition: <| $(conditional_exec) == 1 |> + console_name: consname + doc: Opening the console + key: $(test)_PASS + name: Console creation + steps: + - open: + protocol: terminal + terminal_path: $(test_directory) + - writeln: echo "terminal loaded" + - console: + condition: <| $(conditional_exec) == 1 |> + console_name: consname + key: $(test)_PASS + name: Console read_until with timeout + steps: + - read_until: + expected: terminal loaded + timeout: 5 + - console: + condition: <| $(conditional_exec) == 1 |> + console_name: consname + key: $(test)_PASS + name: Console write + steps: + - writeln: echo 0 + - sleep: + condition: <| $(conditional_exec) == 1 |> + name: sleep item + timeout: 1 + - console: + condition: <| $(conditional_exec) == 1 |> + console_name: consname + key: $(test)_PASS + name: Console read_until immediate + steps: + - read_until: + expected: '0' + timeout: 0 + - console: + condition: <| $(conditional_exec) == 1 |> + console_name: consname + key: $(test)_PASS + name: Console read_until immediate (2) + steps: + - read_until: + expected: $(terminal_prompt) + timeout: 0 + - console: + condition: <| $(conditional_exec) == 1 |> + console_name: consname + key: $(test)_PASS + name: Console closure + steps: + - close: consname + - sleep: + condition: <| $(conditional_exec) == 2 |> + name: sleep item + timeout: 1 + filename: /home/renish/workspace/testium/code/test/validation/items/common/conditional/test.tum + - group: + name: Various syntax robustness + steps: + - sequence: + steps: + - sleep: + key: $(test)_PASS + timeout: 0.2 + - sleep: + key: $(test)_PASS + name: null + timeout: 0.2 + - sleep: + doc: null + key: $(test)_PASS + name: Empty "doc:" declared (must PASS) + timeout: 0.2 + filename: /home/renish/workspace/testium/code/test/validation/items/common/syntax_robustness/test.tum + - group: + name: Helper lib functions + steps: + - py_func: + file: $(test_path)$(psep)helper_lib.py + func_name: check_os + key: $(test)_PASS + name: OS + param: + - $(os) + - py_func: + file: $(test_path)$(psep)helper_lib.py + func_name: check_get_main_dir + key: $(test)_PASS + name: get_main_dir + - py_func: + file: $(test_path)$(psep)helper_lib.py + func_name: check_timestamp_as_sec_conversion + key: $(test)_PASS + name: timestamp_as_sec conversion + - py_func: + file: $(test_path)$(psep)helper_lib.py + func_name: check_timestamp + key: $(test)_PASS + name: timestamp and timestamp_as_sec + filename: /home/renish/workspace/testium/code/test/validation/items/common/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/console.tum b/schema/test_schema/console.tum new file mode 100644 index 0000000..861fa35 --- /dev/null +++ b/schema/test_schema/console.tum @@ -0,0 +1,164 @@ +config_file: +- param.yaml +- items/console/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: console test constants + values: + - test: console + - test_path: items/$(test) + - group: + name: console test + steps: + - sequence: + steps: + - console: + console_name: term + doc: Opening the console + key: $(test)_PASS + name: Console creation + steps: + - open: + protocol: terminal + terminal_path: $(test_directory) + - writeln: echo "endOfOpen" + - console: + console_name: term + key: $(test)_PASS + name: Console read_until with timeout + steps: + - read_until: + expected: endOfOpen + timeout: 5 + - console: + console_name: term + key: $(test)_PASS + name: Console write + steps: + - writeln: echo 0 + - sleep: + name: sleep item + timeout: 1 + - console: + console_name: term + key: $(test)_PASS + name: Console read_until immediate + steps: + - read_until: + expected: '0' + timeout: 0 + - console: + console_name: term + key: $(test)_PASS + name: Console write + steps: + - writeln: echo "HelloConsole" + - console: + console_name: term + key: $(test)_FAIL + name: Console read_until fail + steps: + - read_until: + expected: Something never prints + timeout: 1 + - console: + console_name: term + key: $(test)_PASS + name: Console write + steps: + - writeln: echo "HelloConsole" + - console: + console_name: term + key: $(test)_PASS + name: Console read_until no_fail + steps: + - read_until: + expected: Something never prints + no_fail: true + timeout: 1 + - console: + console_name: term + key: $(test)_PASS + name: Console read_until muted + steps: + - writeln: echo "HelloConsole" + - read_until: + expected: HelloConsole + mute: true + timeout: 1 + - console: + console_name: term + key: $(test)_PASS + name: Console read_until muted + steps: + - writeln: echo "HelloConsole is PASS" && echo "endOfCmd" + - read_until: + expected: endOfCmd + process_result: '''Hello'' in r''''''$(result)'''''' + and ''PASS'' in r''''''$(result)'''''' ' + timeout: 1 + - console: + console_name: term + execute_on_stop: true + key: $(test)_PASS + name: Console closure + steps: + - close: term + filename: /home/renish/workspace/testium/code/test/validation/items/console/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/cycle.tum b/schema/test_schema/cycle.tum new file mode 100644 index 0000000..3d73c0e --- /dev/null +++ b/schema/test_schema/cycle.tum @@ -0,0 +1,106 @@ +config_file: +- param.yaml +- items/cycle/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + terminal_prompt: $(linux_prompt) + psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: cycle test constants + values: + - test: cycle + - test_path: items/$(test) + - group: + name: cycle test + steps: + - sequence: + steps: + - loop: + iterator: 10 + key: $(test)_PASS + name: Cycle number of loops + steps: + - py_func: + file: $(test_path)$(psep)cycle.py + func_name: donothing + name: do nothing + - loop: + iterator: + - 12 + - 20 + - 30 + key: $(test)_PASS + name: Cycle iterating on list + steps: + - py_func: + file: $(test_path)$(psep)cycle.py + func_name: checkloopparam + name: check loop param + param: + - $(loop_param) + - loop: + exit_condition: + file: $(test_path)$(psep)cycle.py + func_name: exitcondition + key: $(test)_PASS + name: Infinite loop with exit condition + steps: + - py_func: + file: $(test_path)$(psep)cycle.py + func_name: donothing + name: do nothing + filename: /home/renish/workspace/testium/code/test/validation/items/cycle/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/dialogs.tum b/schema/test_schema/dialogs.tum new file mode 100644 index 0000000..fa272ee --- /dev/null +++ b/schema/test_schema/dialogs.tum @@ -0,0 +1,135 @@ +config_file: +- param.yaml +- items/dialogs/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: dialogs test constants + values: + test: dialogs + test_path: items/$(test) + - group: + name: dialogs test + steps: + - sequence: + steps: + - dialog_image: + auto_result: ok + condition: $(validation_dialogs) + filename: $(test_path)$(psep)IMG_20140213_171455.jpg + key: $(test)_PASS + name: dialog image PASS + question: click ok if you see the image + - dialog_image: + auto_result: cancel + condition: $(validation_dialogs) + filename: $(test_path)$(psep)IMG_20140213_171455.jpg + key: $(test)_FAIL + name: dialog image FAIL + question: click cancel + - dialog_references: + auto_result: ok + condition: $(validation_dialogs) + key: $(test)_PASS + name: dialog_reference PASS + question: click ok + - dialog_references: + auto_result: cancel + condition: $(validation_dialogs) + key: $(test)_FAIL + name: dialog_reference FAIL + question: click cancel + - dialog_value: + auto_result: ok + auto_value: '123' + condition: $(validation_dialogs) + key: $(test)_PASS + name: dialog_value PASS + question: enter 123 and click ok + - dialog_value: + auto_result: ok + condition: $(validation_dialogs) + key: $(test)_FAIL + name: dialog_value empty FAIL + question: enter nothing and click ok + - dialog_value: + auto_result: cancel + condition: $(validation_dialogs) + key: $(test)_FAIL + name: dialog_value canceled FAIL + question: enter nothing and click cancel + - dialog_message: + auto_result: ok + condition: $(validation_dialogs) + key: $(test)_PASS + name: dialog_message PASS + question: click ok + - dialog_question: + auto_result: 'yes' + condition: $(validation_dialogs) + key: $(test)_PASS + name: dialog_question PASS + question: click yes + - dialog_question: + auto_result: 'no' + condition: $(validation_dialogs) + key: $(test)_FAIL + name: dialog_question FAIL + question: click no + filename: /home/renish/workspace/testium/code/test/validation/items/dialogs/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/expanse.tum b/schema/test_schema/expanse.tum new file mode 100644 index 0000000..195e7eb --- /dev/null +++ b/schema/test_schema/expanse.tum @@ -0,0 +1,130 @@ +config_file: +- param.yaml +- items/expanse/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: expanse test constants + values: + test: expanse + test_path: items/$(test) + - group: + name: expanse test + steps: + - sequence: + steps: + - check: + key: $(test)_PASS + name: Check variables expansion is correct (PASS) + values: + - <| $(expanse_index) == 1 |> + - <| $(expanse_table)[$(expanse_index)] == 9012 |> + - <| $(expanse_eval) == True |> + - let: + key: $(test)_PASS + name: Dynamic variables expansion + values: + - expanse_select: <|"$(expanse_select)".replace("o", "a")|> + - expanse_index: $(expanse_index_$(expanse_select)) + - expanse_table: $(expanse_table_$(expanse_select)) + - expanse_eval: <|$(expanse_index) == 1|> + - check: + key: $(test)_PASS + name: Check variables expansion is correct (PASS) + values: + - <| $(expanse_index) == 0 |> + - <| $(expanse_table)[$(expanse_index)] == "abcd" |> + - <| $(expanse_eval) == False |> + - let: + key: $(test)_PASS + name: Complex variables expansion + values: + - var1: expanse + - var2: var + - var3: bla + - var4: blo + - expanse_var_bla: 3 + - expanse_blo_var: 5 + - expanse_complex: <|<|$(expanse_$(var2)_$(var3))*6|> + + <|4*$($(var1)_$(var4)_$(var2))|>|> + - check: + key: $(test)_PASS + name: Check complex variables expansion is correct (PASS) + values: + - <| $(expanse_complex) == 38 |> + - let: + key: $(test)_PASS + name: Variables expansion in object + values: + - expanse_key: b + - expanse_var: 3 + - expanse_var_2: 6 + - expanse_object: + - $(expanse_key): <|2**3|> + a: $(expanse_var_2) + - <|"bla".replace("a", "o")|>: + - <|$(expanse_var)*$(expanse_var_2)|> + - 25 + - check: + key: $(test)_PASS + name: Check complex variables expansion is correct (PASS) + values: + - '<| $(expanse_object) == [{"a": 6, "b": 8}, {"blo": [18, + 25]}] |>' + filename: /home/renish/workspace/testium/code/test/validation/items/expanse/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/git.tum b/schema/test_schema/git.tum new file mode 100644 index 0000000..8553a15 --- /dev/null +++ b/schema/test_schema/git.tum @@ -0,0 +1,82 @@ +config_file: +- param.yaml +- items/git/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: git test constants + values: + test: git + test_path: items/$(test) + - group: + name: git test + steps: + - sequence: + steps: + - git: + key: $(test)_PASS + name: Testium repo + repo: $(test_directory) + - git: + key: $(test)_PASS + name: Testium repo + repo: + - $(test_directory) + - $(test_directory) + filename: /home/renish/workspace/testium/code/test/validation/items/git/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/include.tum b/schema/test_schema/include.tum new file mode 100644 index 0000000..d640184 --- /dev/null +++ b/schema/test_schema/include.tum @@ -0,0 +1,129 @@ +config_file: +- param.yaml +- items/include/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: include test constants + values: + test: include + test_path: items/$(test) + - group: + name: include test + steps: + - sequence: + steps: + - sequence: + steps: + - py_func: + file: $(test_path)$(psep)include.py + func_name: ValidationTest + key: $(test)_PASS + name: My first include test + param: + - $(test parameter) + filename: /home/renish/workspace/testium/code/test/validation/items/include/inc + no template/my first include.tum + - sequence: + steps: + - py_func: + file: $(test_path)$(psep)include.py + func_name: ValidationTest + key: $(test)_PASS + name: My first include test + param: + - $(test parameter) + filename: /home/renish/workspace/testium/code/test/validation/items/include/inc + no template/my first include.tum + - sequence: + steps: + - py_func: + file: $(test_path)$(psep)include.py + func_name: ValidationTest + key: $(test)_PASS + name: My second include test + param: + - My second include test parameter + filename: /home/renish/workspace/testium/code/test/validation/items/include/inc + with template/my second include.tum + - sequence: + steps: + - py_func: + file: $(test_path)$(psep)include.py + func_name: ValidationTest + key: $(test)_PASS + name: My second include test + param: + - My second include test parameter + filename: /home/renish/workspace/testium/code/test/validation/items/include/inc + with template/my second include.tum + - let: + name: Declare param for inclusion + values: + - inc: Dali + - Dali_inc: Dalida + - sequence: + steps: + - let: + name: Test param inclusion 1 + values: + - inclusion: $($(inc)_inc) + filename: /home/renish/workspace/testium/code/test/validation/items/include/inc + with template/my_3d_include.tum + filename: /home/renish/workspace/testium/code/test/validation/items/include/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/isolation.tum b/schema/test_schema/isolation.tum new file mode 100644 index 0000000..389b483 --- /dev/null +++ b/schema/test_schema/isolation.tum @@ -0,0 +1,80 @@ +config_file: +- param.yaml +- items/isolation/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: isolation test constants + values: + test: isolation + test_path: items/$(test) + - group: + name: isolation test + steps: + - sequence: + steps: + - py_func: + expected_result: true + file: $(test_path)$(psep)check_isolation.py + func_name: check_isolation + key: $(test)_PASS + name: py_func/lua_func do not depend on testium internals + param: + - $(testium_path) + filename: /home/renish/workspace/testium/code/test/validation/items/isolation/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/jsonrpc.tum b/schema/test_schema/jsonrpc.tum new file mode 100644 index 0000000..02a407b --- /dev/null +++ b/schema/test_schema/jsonrpc.tum @@ -0,0 +1,490 @@ +config_file: +- param.yaml +- items/jsonrpc/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: jsonrpc test constants + values: + test: jsonrpc + test_path: items/$(test) + - group: + name: jsonrpc test + steps: + - sequence: + steps: + - console: + console_name: jrpces + doc: check if jrpc_echo_server.py is available + key: $(test)_PASS + name: json rpc echo server + steps: + - open: + protocol: terminal + - read_until: + expected: $(terminal_prompt) + no_fail: true + timeout: 1 + - writeln: test -f /home/renish/workspace/testium/code/test/validation/items/jsonrpc/jrpc_echo_server.py + && echo JRPC_OK + - read_until: + expected: JRPC_OK + no_fail: true + timeout: 2 + - group: + condition: <| 'JRPC_OK' in r'''$(cn_json rpc echo server)''' |> + name: jsonrpc tests + steps: + - console: + console_name: jrpces + key: $(test)_PASS + name: Start the json rpc echo server + steps: + - writeln: python3 /home/renish/workspace/testium/code/test/validation/items/jsonrpc/jrpc_echo_server.py + -c /home/renish/workspace/testium/code/test/validation/items/jsonrpc/jrpces.ini + - read_until: + expected: ready + timeout: 5 + - console: + console_name: jsonrpc_server + doc: Opening the RAW TCP console + key: $(test)_PASS + name: Open the raw tcp Console + skipped: $(skip_tcp) + steps: + - open: + protocol: rawtcp + tcp_host: localhost + tcp_port: 4321 + - json_rpc: + console: + name: jsonrpc_server + key: $(test)_PASS + name: JSONRPC console Query waiting for reception + skipped: $(skip_tcp) + steps: + - query: + method: echo + params: + - Hello World + - a: 1 + b: hello + timeout: 1 + - json_rpc: + console: + name: jsonrpc_server + key: $(test)_PASS + name: JSONRPC console Query not waiting (only send) + skipped: $(skip_tcp) + steps: + - query: + id: 3095372 + method: echo + no_wait: true + params: + - a: -1 + b: olleh + - World Hello + timeout: 1 + - sleep: + name: Small delay for the test + skipped: $(skip_tcp) + timeout: 1 + - json_rpc: + console: + name: jsonrpc_server + key: $(test)_PASS + name: JSONRPC console Reception + skipped: $(skip_tcp) + steps: + - receive: + id: 3095372 + timeout: 1 + - console: + console_name: jsonrpc_server + doc: Opening the RAW TCP console + execute_on_stop: true + key: $(test)_PASS + name: Close the raw tcp console + skipped: $(skip_tcp) + steps: + - close: null + - json_rpc: + key: $(test)_PASS + name: JSONRPC UDP query waiting for reception + steps: + - open: null + - query: + method: echo + name: echo + params: + - Hello World + - a: 1 + b: hello + timeout: 1 + - close: null + timeout: 1 + udp: + rcv_port: 8765 + server: localhost + snd_port: 4323 + - json_rpc: + key: $(test)_FAIL + name: Failing JSONRPC UDP query waiting for reception + (returning an error) + steps: + - open: null + - query: + method: echo2 + params: + - Hello World + - a: 1 + b: hello + timeout: 1 + - close: null + timeout: 1 + udp: + rcv_port: 8765 + server: localhost + snd_port: 4323 + - json_rpc: + key: $(test)_PASS + name: JSONRPC UDP query waiting for reception of + an expected error + steps: + - open: null + - query: + expected_result: + code: -32000 + message: function not found + method: echo2 + params: + - Hello World + - a: 1 + b: hello + - close: null + timeout: 1 + udp: + rcv_port: 8765 + server: localhost + snd_port: 4323 + - json_rpc: + doc: 'Failing JSONRPC UDP query waiting for reception + and checking result + + and timeout elapses (wrong udp port) + + ' + key: $(test)_FAIL + name: Failing UDP JSONRPC query timeout elapses (wrong + udp port) + steps: + - open: null + - query: + method: echo + params: + - Hello World + - a: 1 + b: hello + timeout: 0.5 + - close: null + timeout: 1 + udp: + rcv_port: 48393 + server: localhost + snd_port: 4326 + - json_rpc: + key: $(test)_PASS + name: JSONRPC UDP query not waiting (only send) + steps: + - open: null + - query: + id: 3095372 + method: echo + no_wait: true + params: + - a: -1 + b: olleh + - World Hello + timeout: 1 + udp: + rcv_port: 8765 + server: localhost + snd_port: 4323 + - sleep: + name: Small delay for the test + timeout: 1 + - json_rpc: + key: $(test)_PASS + name: JSONRPC UDP Reception + steps: + - receive: + id: 3095372 + - close: null + timeout: 1 + udp: + rcv_port: 8765 + server: localhost + snd_port: 4323 + - json_rpc: + key: $(test)_PASS + name: JSONRPC UDP query not waiting (only send) + steps: + - open: null + - query: + id: 3095372 + method: echo2 + no_wait: true + params: + - a: -1 + b: olleh + - World Hello + timeout: 1 + udp: + rcv_port: 8765 + server: localhost + snd_port: 4323 + - sleep: + name: Small delay for the test + timeout: 1 + - json_rpc: + key: $(test)_FAIL + name: Failing JSONRPC UDP Reception (returning an + error) + steps: + - receive: + id: 3095372 + timeout: 1 + - close: null + timeout: 1 + udp: + rcv_port: 8765 + server: localhost + snd_port: 4323 + - json_rpc: + doc: JSONRPC UDP query waiting for reception and + checking result + key: $(test)_PASS + name: UDP JSONRPC query waiting and checking + steps: + - open: null + - query: + expected_result: + - - Hello World + - a: 1 + b: hello + - {} + method: echo + params: + - Hello World + - a: 1 + b: hello + timeout: 1 + - close: null + timeout: 1 + udp: + rcv_port: 48393 + server: localhost + snd_port: 4323 + - json_rpc: + doc: JSONRPC UDP query waiting for reception and + checking result + key: $(test)_FAIL + name: Failing UDP JSONRPC query waiting and checking + steps: + - open: null + - query: + expected_result: + - [] + - {} + method: echo + params: + - Hello World + - a: 1 + b: hello + timeout: 1 + - close: null + timeout: 1 + udp: + rcv_port: 48393 + server: localhost + snd_port: 4323 + - json_rpc: + doc: JSONRPC UDP query not waiting, with the purpose + to check the result at reception + key: $(test)_PASS + name: UDP JSONRPC query not waiting (for checking) + steps: + - open: null + - query: + id: 3095372 + method: echo + no_wait: true + params: + - a: -1 + b: olleh + - World Hello + timeout: 1 + udp: + rcv_port: 9876 + server: localhost + snd_port: 4323 + - sleep: + name: Small delay for the test + timeout: 1 + - json_rpc: + doc: JSONRPC UDP Reception and checking result + key: $(test)_PASS + name: UDP JSONRPC reception checking + steps: + - receive: + expected_result: + - - a: -1 + b: olleh + - World Hello + - {} + id: 3095372 + timeout: 1 + - close: null + timeout: 1 + udp: + rcv_port: 9876 + server: localhost + snd_port: 4323 + - json_rpc: + doc: JSONRPC UDP Reception and checking result + key: $(test)_FAIL + name: Failing UDP JSONRPC reception checking + steps: + - receive: + expected_result: + - - a: -1 + b: ollhe + - World Hello + - {} + id: 3095372 + timeout: 1 + - close: null + timeout: 1 + udp: + rcv_port: 9876 + server: localhost + snd_port: 4323 + - json_rpc: + doc: 'JSONRPC UDP query waiting for reception and + checking result with + + replacing $(result) and evaluating string. + + ' + key: $(test)_PASS + name: UDP JSONRPC query waiting and evaluating result + steps: + - open: null + - query: + expected_result: 1 + method: echo + params: + - Hello World + - a: 1 + b: hello + process_result: $(result)[0][1]['a'] + timeout: 1 + - close: null + timeout: 1 + udp: + rcv_port: 48393 + server: localhost + snd_port: 4323 + - json_rpc: + doc: 'JSONRPC UDP query waiting for reception and + checking result with + + replacing $(result) and evaluating string. + + ' + key: $(test)_FAIL + name: Failing UDP JSONRPC query waiting and evaluating + result + steps: + - open: null + - query: + expected_result: $(result)[0][1]['a'] == 0 + method: echo + params: + - Hello World + - a: 1 + b: hello + timeout: 1 + - close: null + timeout: 1 + udp: + rcv_port: 48393 + server: localhost + snd_port: 4323 + - console: + console_name: jrpces + doc: check if the jsonrpc echo server is installed + execute_on_stop: true + key: $(test)_PASS + name: Stop json rpc echo server + steps: + - close: + protocol: terminal + filename: /home/renish/workspace/testium/code/test/validation/items/jsonrpc/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/let.tum b/schema/test_schema/let.tum new file mode 100644 index 0000000..618678d --- /dev/null +++ b/schema/test_schema/let.tum @@ -0,0 +1,157 @@ +config_file: +- param.yaml +- items/let/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: let test constants + values: + - test: let + - test_path: items/$(test) + - group: + name: let test + steps: + - sequence: + steps: + - loop: + iterator: 10 + key: $(test)_PASS + name: Cycle number of loops + steps: + - py_func: + file: $(test_path)$(psep)let.py + func_name: donothing + name: do nothing + - let: + name: Let it be + values: + - be: <| $(loop_param) == $(it) |> + - it: $(loop_param) + - loop: + iterator: + - 12 + - 20 + - 30 + key: $(test)_PASS + name: Cycle iterating on list + steps: + - py_func: + file: $(test_path)$(psep)let.py + func_name: checkloopparam + name: check loop param + param: + - $(loop_param) + - let: + name: Let it be + values: + - it: $(loop_param) + - be: <| $(loop_param) == $(it) |> + - let: + key: $(test)_PASS + name: Get time + values: + - loop_t0: $(ts_start_Cycle iterating on list) + - loop_t1: $(ts_end_Cycle iterating on list) + - loop_duration: $(ts_duration_Cycle iterating on list) + - let: + key: $(test)_PASS + name: Get parameter file value + values: + - test_overwrite_me: <| $(overwrite_me) == True |> + - py_func: + file: $(test_path)$(psep)let.py + func_name: checkGlobalDic + name: Check global dic pass + param: + - test_overwrite_me + - true + - let: + key: $(test)_PASS + name: Overwrite parameter file value + values: + - overwrite_me: false + - py_func: + expected_result: $(overwrite_me) == False + file: $(test_path)$(psep)let.py + func_name: checkGlobalDic + key: $(test)_FAIL + name: Check global dic fail + param: + - overwrite_me + - true + - py_func: + expected_result: fail + file: $(test_path)$(psep)let.py + func_name: checkGlobalDic + key: $(test)_PASS + name: Check global dic fail + param: + - overwrite_me + - true + - let: + key: $(test)_PASS + name: Evaluate Overwriting parameter file value + values: + - test_overwrite_me: <| "$(overwrite_me)" == True |> + - check: + key: $(test)_PASS + name: Check Overwriting parameter file value + values: + - <| $(test_overwrite_me) == False |> + filename: /home/renish/workspace/testium/code/test/validation/items/let/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/lua_func.tum b/schema/test_schema/lua_func.tum new file mode 100644 index 0000000..b4b0240 --- /dev/null +++ b/schema/test_schema/lua_func.tum @@ -0,0 +1,297 @@ +config_file: +- param.yaml +- items/lua_func/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: lua_func test constants + values: + test: lua_func + test_path: items/$(test) + - group: + name: lua_func test + steps: + - sequence: + steps: + - let: + name: lua_func test constants, + values: + lua_func test parameter: test parameter lua_func + - lua_func: + file: $(test_path)$(psep)lua_func.lua + func_name: assertparam + key: $(test)_FAIL + name: fail lua_func + param: + - false + - lua_func: + expected_result: FAIL + file: $(test_path)$(psep)lua_func.lua + func_name: assertparam + key: $(test)_PASS + name: fail lua_func with expected result FAIL + param: + - false + - lua_func: + expected_result: FAIL + file: $(test_path)$(psep)lua_func.lua + func_name: assertparam + key: $(test)_FAIL + name: pass lua_func with expected result FAIL + param: + - true + - lua_func: + expected_result: -1 + file: $(test_path)$(psep)lua_func.lua + func_name: echo + key: $(test)_PASS + name: expected -1 + param: + - -1 + - lua_func: + expected_result: 354848436 - 354848437 + file: $(test_path)$(psep)lua_func.lua + func_name: echo + key: $(test)_PASS + name: expected eval + param: + - -1 + - lua_func: + expected_result: '[-1, ''a'', {''toto'': ''tata''}]' + file: $(test_path)$(psep)lua_func.lua + func_name: echo + key: $(test)_PASS + name: expected table + param: + - - -1 + - a + - toto: tata + - lua_func: + expected_result: $(lua_func test parameter) + file: $(test_path)$(psep)lua_func.lua + func_name: checkglobal + key: $(test)_PASS + name: global param lua_func + param: + - lua_func test parameter + - lua_func: + expected_result: ($(lua_data_to_be_returned))[0] + file: $(test_path)$(psep)lua_func.lua + func_name: checkglobal2 + key: $(test)_PASS + name: global param lua_func 1 + param: + - 1 + - lua_func: + expected_result: ($(lua_data_to_be_returned))[1] + file: $(test_path)$(psep)lua_func.lua + func_name: checkglobal2 + key: $(test)_PASS + name: global param lua_func 2 + param: + - 2 + - lua_func: + expected_result: ($(lua_data_to_be_returned))[2] + file: $(test_path)$(psep)lua_func.lua + func_name: checkglobal2 + key: $(test)_PASS + name: global param lua_func 3 + param: + - 3 + - let: + key: $(test)_PASS + name: python2func + values: + - py: $(test_path)$(psep)lua_func.lua + - lua_func: + file: $(test_path)$(psep)lua_func.lua + func_name: should_not_be_called + name: skipped_checkglobal + param: + - $(test parameter) + - lua_func: + file: $(test_path)$(psep)lua_func.lua + func_name: checkglobal + name: skipped true + param: + - $(test parameter) + skipped: true + - lua_func: + file: $(test_path)$(psep)lua_func.lua + func_name: checkglobal + name: skipped 1 + param: + - $(test parameter) + skipped: true + - group: + name: Function results check + steps: + - group: + name: Function result failure + steps: + - lua_func: + file: $(test_path)$(psep)lua_func.lua + func_name: echo + key: $(test)_PASS + name: int failure + param: + - -1 + - lua_func: + file: $(test_path)$(psep)lua_func.lua + func_name: echo + key: $(test)_PASS + name: float failure + param: + - -1.3 + - lua_func: + file: $(test_path)$(psep)lua_func.lua + func_name: echo + key: $(test)_PASS + name: String failure + param: + - FAIL + - lua_func: + file: $(test_path)$(psep)lua_func.lua + func_name: tuple_return + key: $(test)_PASS + name: Tuple int,str failure + param: + - -1 + - Got a failure + - group: + name: Functions result success + steps: + - lua_func: + file: $(test_path)$(psep)lua_func.lua + func_name: echo + key: $(test)_PASS + name: int success + param: + - 0 + - lua_func: + file: $(test_path)$(psep)lua_func.lua + func_name: echo + key: $(test)_PASS + name: float success + param: + - 0.3 + - lua_func: + file: $(test_path)$(psep)lua_func.lua + func_name: echo + key: $(test)_PASS + name: String success + param: + - Something that is not only strictly FAIL + - lua_func: + file: $(test_path)$(psep)lua_func.lua + func_name: tuple_return + key: $(test)_PASS + name: Tuple int,str success + param: + - 0 + - OK + - lua_func: + file: $(test_path)$(psep)lua_func.lua + func_name: test_delgd + key: $(test)_PASS + name: delgd test + - lua_func: + file: $(test_path)$(psep)lua_func.lua + func_name: return_nothing + key: $(test)_PASS + name: function returning nothing should succeed + - lua_func: + file: $(test_path)$(psep)lua_func.lua + func_name: return_explicit_nil + key: $(test)_PASS + name: function returning explicit nil should succeed + - group: + name: context_id tests + steps: + - lua_func: + context_id: lua_ctx_test + expected_result: hello lua + file: $(test_path)$(psep)lua_func.lua + func_name: set_context_value + key: $(test)_PASS + name: set context value + param: + - hello lua + - lua_func: + context_id: lua_ctx_test + expected_result: hello lua + file: $(test_path)$(psep)lua_func.lua + func_name: get_context_value + key: $(test)_PASS + name: get context value (same context_id) + - lua_func: + expected_result: hello lua + file: $(test_path)$(psep)lua_func.lua + func_name: get_context_value + key: $(test)_PASS + name: get context value (no context_id, from main + gd) + - lua_func: + context_id: lua_ctx_other + expected_result: hello lua + file: $(test_path)$(psep)lua_func.lua + func_name: get_context_value + key: $(test)_PASS + name: get context value (different context_id) + filename: /home/renish/workspace/testium/code/test/validation/items/lua_func/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/parallel.tum b/schema/test_schema/parallel.tum new file mode 100644 index 0000000..9eef252 --- /dev/null +++ b/schema/test_schema/parallel.tum @@ -0,0 +1,414 @@ +config_file: +- param.yaml +- items/parallel/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: parallel test constants + values: + test: parallel + test_path: items/$(test) + - group: + name: parallel test + steps: + - sequence: + steps: + - parallel: + branches: + - name: Branch A + steps: + - let: + name: Set A done + values: + - branch_a_done: true + - name: Branch B + steps: + - let: + name: Set B done + values: + - branch_b_done: true + key: $(test)_PASS + name: Both branches pass + sync: all + - check: + key: $(test)_PASS + name: Both branches ran + values: + - <| $(branch_a_done) == True |> + - <| $(branch_b_done) == True |> + - parallel: + branches: + - name: Pass branch + steps: + - let: + name: Set pass flag + values: + - pass_branch_ran: true + - name: Fail branch + steps: + - py_func: + expected_result: fail + file: $(test_path)$(psep)parallel.py + func_name: sleep_func + name: Raise exception + param: + - 0 + key: $(test)_PASS + name: One branch fails + no_fail: true + sync: all + - check: + key: $(test)_PASS + name: Pass branch still ran + values: + - <| $(pass_branch_ran) == True |> + - let: + name: Reset slow flag + values: + - slow_done: false + - parallel: + branches: + - name: Fast branch + steps: + - let: + name: Fast done + values: + - fast_done: true + - name: Slow branch + steps: + - py_func: + file: $(test_path)$(psep)parallel.py + func_name: sleep_func + name: Sleep 2s + param: + - 2 + - let: + name: Slow done + values: + - slow_done: true + key: $(test)_PASS + name: sync any - first wins + sync: any + - check: + key: $(test)_PASS + name: Fast branch ran, slow branch was stopped + values: + - <| $(fast_done) == True |> + - <| $(slow_done) == False |> + - let: + name: Reset sync flag + values: + - sync_flag: '' + - waiter_ran: false + - parallel: + branches: + - name: Setter branch + steps: + - py_func: + file: $(test_path)$(psep)parallel.py + func_name: sleep_func + name: Sleep 0.3s then set flag + param: + - 0.3 + - let: + name: Set sync flag + values: + - sync_flag: ready + - name: Waiter branch + steps: + - let: + name: Got flag + values: + - waiter_ran: true + wait_for: + condition: <| "$(sync_flag)" == "ready" |> + timeout: 10 + key: $(test)_PASS + name: wait_for synchronization + sync: all + - check: + key: $(test)_PASS + name: Waiter branch ran after flag was set + values: + - <| $(waiter_ran) == True |> + - parallel: + branches: + - name: Sleep A + steps: + - sleep: + name: Sleep 1s A + timeout: 1 + - name: Sleep B + steps: + - sleep: + name: Sleep 1s B + timeout: 1 + key: $(test)_PASS + name: Timing test + sync: all + - let: + name: Capture parallel duration + values: + - parallel_duration: $(ts_duration_Timing test) + - check: + key: $(test)_PASS + name: Duration < 1.8s (would be 2s if sequential) + values: + - <| float("$(parallel_duration)") < 1.8 |> + - let: + name: Reset N flags + values: + - n_a: false + - n_b: false + - n_c: false + - n_d: false + - parallel: + branches: + - name: NA + steps: + - let: + name: set n_a + values: + - n_a: true + - name: NB + steps: + - let: + name: set n_b + values: + - n_b: true + - name: NC + steps: + - let: + name: set n_c + values: + - n_c: true + - name: ND + steps: + - let: + name: set n_d + values: + - n_d: true + key: $(test)_PASS + name: Four branches + sync: all + - check: + key: $(test)_PASS + name: Four branches all set their flag + values: + - <| $(n_a) == True |> + - <| $(n_b) == True |> + - <| $(n_c) == True |> + - <| $(n_d) == True |> + - let: + name: Reset nested flags + values: + - outer_x: false + - inner_x_1: false + - inner_x_2: false + - parallel: + branches: + - name: Outer X + steps: + - let: + name: set outer_x + values: + - outer_x: true + - parallel: + branches: + - name: Inner X1 + steps: + - let: + name: set inner_x_1 + values: + - inner_x_1: true + - name: Inner X2 + steps: + - let: + name: set inner_x_2 + values: + - inner_x_2: true + name: Inner parallel + sync: all + - name: Outer Y + steps: + - sleep: + name: brief sleep + timeout: 0 + key: $(test)_PASS + name: Outer parallel + sync: all + - check: + key: $(test)_PASS + name: Nested parallel set all flags + values: + - <| $(outer_x) == True |> + - <| $(inner_x_1) == True |> + - <| $(inner_x_2) == True |> + - let: + name: Reset waiter timeout flag + values: + - waiter_timeout_ran: false + - parallel: + branches: + - name: Quick branch + steps: + - sleep: + name: brief sleep + timeout: 0 + - name: Doomed waiter + steps: + - let: + name: should not run + values: + - waiter_timeout_ran: true + wait_for: + condition: <| "never" == "ready" |> + timeout: 1 + key: $(test)_PASS + name: wait_for timeout + no_fail: true + sync: all + - check: + key: $(test)_PASS + name: Doomed waiter never ran its steps + values: + - <| $(waiter_timeout_ran) == False |> + - parallel: + branches: + - name: ok branch + steps: + - let: + name: noop + values: + - noop_var: 1 + - name: broken branch + steps: + - py_func: + expected_result: fail + file: $(test_path)$(psep)parallel.py + func_name: sleep_func + name: Forced fail + param: + - 0 + key: $(test)_FAIL + name: One branch really fails + sync: all + - let: + name: Reset branch condition flag + values: + - cond_branch_ran: false + - other_branch_ran: false + - parallel: + branches: + - condition: <| "always" == "false" |> + name: Skipped branch + steps: + - let: + name: should not run + values: + - cond_branch_ran: true + - name: Other branch + steps: + - let: + name: ran + values: + - other_branch_ran: true + key: $(test)_PASS + name: Condition-skipped branch + sync: all + - check: + key: $(test)_PASS + name: Skipped condition branch did not run + values: + - <| $(cond_branch_ran) == False |> + - <| $(other_branch_ran) == True |> + - let: + name: Reset loop counters + values: + - loop_count_a: 0 + - loop_count_b: 0 + - loop: + iterator: 3 + name: Loop wrapping parallel + steps: + - parallel: + branches: + - name: LA + steps: + - let: + name: bump A + values: + - loop_count_a: <| int("$(loop_count_a)") + + 1 |> + - name: LB + steps: + - let: + name: bump B + values: + - loop_count_b: <| int("$(loop_count_b)") + + 1 |> + name: Per-iteration parallel + sync: all + - check: + key: $(test)_PASS + name: Both branches ran 3 times + values: + - <| int("$(loop_count_a)") == 3 |> + - <| int("$(loop_count_b)") == 3 |> + filename: /home/renish/workspace/testium/code/test/validation/items/parallel/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/plot.tum b/schema/test_schema/plot.tum new file mode 100644 index 0000000..7a08667 --- /dev/null +++ b/schema/test_schema/plot.tum @@ -0,0 +1,137 @@ +config_file: +- param.yaml +- items/plot/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: plot test constants + values: + test: plot + test_path: items/$(test) + - group: + name: plot test + steps: + - sequence: + steps: + - group: + condition: <| $(validation_dialogs) and not tm.text_mode() + |> + name: Plot test + steps: + - plot: + key: $(test)_PASS + name: Open the plot + plot_name: Mon Plot + steps: + - open: + log_path: $(validation_report_path) + - plot: + key: $(test)_PASS + name: Add periodic to the plot + plot_name: Mon Plot + steps: + - periodic: + eval: '{"periodic": $(result)}' + file: $(test_path)$(psep)plot.py + func_name: random_value + period: 1 + - sleep: + dialog: true + name: sleep + timeout: 3 + - loop: + iterator: 10 + name: Add of other data in the plot + steps: + - plot: + key: $(test)_PASS + name: Add to the plot + plot_name: Mon Plot + steps: + - add: + value1: $(loop_index) + value2: $(loop_index)+2 + - sleep: + name: sleep between values + timeout: 1 + - py_func: + file: $(test_path)$(psep)plot.py + func_name: LastValues + key: $(test)_PASS + name: last plot values + param: + - Mon Plot + - plot: + execute_on_stop: true + key: $(test)_PASS + name: Export + plot_name: Mon Plot + steps: + - export: $(validation_report_path)/plot_export.pdf + - export: $(validation_report_path)/plot_export.csv + - plot: + execute_on_stop: true + key: $(test)_PASS + name: Close the plot + plot_name: Mon Plot + steps: + - close: + timeout: 2 + wait_dialog_exit: true + filename: /home/renish/workspace/testium/code/test/validation/items/plot/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/py_func.tum b/schema/test_schema/py_func.tum new file mode 100644 index 0000000..1385ad9 --- /dev/null +++ b/schema/test_schema/py_func.tum @@ -0,0 +1,326 @@ +config_file: +- param.yaml +- items/py_func/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: py_func test constants + values: + test: py_func + test_path: items/$(test) + - group: + name: py_func test + steps: + - sequence: + steps: + - let: + name: py_func test constants, + values: + py_func test parameter: test parameter + - py_func: + file: $(test_path)$(psep)py_func.py + func_name: assertparam + key: $(test)_PASS + name: pass py_func + param: + - true + - py_func: + file: $(test_path)$(psep)py_func.py + func_name: assertparam + key: $(test)_FAIL + name: fail py_func + param: + - false + - py_func: + expected_result: FAIL + file: $(test_path)$(psep)py_func.py + func_name: assertparam + key: $(test)_PASS + name: fail py_func with expected result "FAIL" + param: + - false + - py_func: + expected_result: FAIL + file: $(test_path)$(psep)py_func.py + func_name: assertparam + key: $(test)_FAIL + name: pass py_func with expected result FAIL + param: + - true + - py_func: + expected_result: -1 + file: $(test_path)$(psep)py_func.py + func_name: echo + key: $(test)_PASS + name: expected -1 + param: + - -1 + - py_func: + expected_result: 354848436 - 354848437 + file: $(test_path)$(psep)py_func.py + func_name: echo + key: $(test)_PASS + name: expected eval + param: + - -1 + - py_func: + expected_result: '[-1, ''a'', {''toto'': ''tata''}]' + file: $(test_path)$(psep)py_func.py + func_name: echo + key: $(test)_PASS + name: expected table + param: + - - -1 + - a + - toto: tata + - py_func: + file: $(test_path)$(psep)py_func.py + func_name: checkglobal + key: $(test)_PASS + name: global param py_func + param: + - $(py_func test parameter) + - let: + key: $(test)_PASS + name: python2func + values: + - py: $(test_path)$(psep)py_func.py + - py_func: + expected_result: $(py_func test parameter) + file: $(py) + func_name: checkglobal2 + key: $(test)_PASS + name: global param py_func 2 + - py_func: + file: $(py) + func_name: checkglobal + key: $(test)_PASS + name: global param py_func + param: + - $(py_func test parameter) + - py_func: + file: $(test_path)$(psep)py_func.py + func_name: should_not_be_called + name: skipped_checkglobal + param: + - $(py_func test parameter) + - py_func: + file: $(test_path)$(psep)py_func.py + func_name: checkglobal + name: skipped true + param: + - $(py_func test parameter) + skipped: true + - py_func: + file: $(test_path)$(psep)py_func.py + func_name: checkglobal + name: skipped 1 + param: + - $(py_func test parameter) + skipped: 1 + - py_func: + file: $(test_path)$(psep)py_func.py + func_name: ValidationTest + name: FunctionItem test + param: + - $(py_func test parameter) + - group: + name: Function results check + steps: + - group: + name: Function result 1 + steps: + - py_func: + expected_result: -1 + file: $(test_path)$(psep)py_func.py + func_name: echo + key: $(test)_PASS + name: int failure + param: + - -1 + - py_func: + expected_result: -1.3 + file: $(test_path)$(psep)py_func.py + func_name: echo + key: $(test)_PASS + name: float failure + param: + - -1.3 + - py_func: + expected_result: FAIL + file: $(test_path)$(psep)py_func.py + func_name: echo + key: $(test)_PASS + name: String failure + param: + - FAIL + - py_func: + expected_result: + - -1 + - Got a failure + file: $(test_path)$(psep)py_func.py + func_name: tuple_return + key: $(test)_PASS + name: Tuple int,str failure + param: + - -1 + - Got a failure + - group: + name: Functions result 2 + steps: + - py_func: + expected_result: 0 + file: $(test_path)$(psep)py_func.py + func_name: echo + key: $(test)_PASS + name: int success + param: + - 0 + - py_func: + expected_result: 0.3 + file: $(test_path)$(psep)py_func.py + func_name: echo + key: $(test)_PASS + name: float success + param: + - 0.3 + - py_func: + expected_result: Something that is not only + strictly FAIL + file: $(test_path)$(psep)py_func.py + func_name: echo + key: $(test)_PASS + name: String success + param: + - Something that is not only strictly FAIL + - py_func: + expected_result: + - 0 + - OK + file: $(test_path)$(psep)py_func.py + func_name: tuple_return + key: $(test)_PASS + name: Tuple int,str success + param: + - 0 + - OK + - py_func: + file: $(test_path)$(psep)py_func.py + func_name: test_delgd + key: $(test)_PASS + name: delgd test + - py_func: + file: $(test_path)$(psep)py_func.py + func_name: return_nothing + key: $(test)_PASS + name: function returning nothing should succeed + - py_func: + file: $(test_path)$(psep)py_func.py + func_name: return_explicit_none + key: $(test)_PASS + name: function returning explicit None should succeed + - group: + name: context_id tests + steps: + - py_func: + expected_result: hello context + file: $(test_path)$(psep)py_func.py + func_name: set_context_value + key: $(test)_PASS + name: set serializable value + param: + - hello context + - py_func: + context_id: ctx_test + expected_result: hello context + file: $(test_path)$(psep)py_func.py + func_name: get_context_value + key: $(test)_PASS + name: get serializable value (same context_id) + - py_func: + expected_result: hello context + file: $(test_path)$(psep)py_func.py + func_name: get_context_value + key: $(test)_PASS + name: get serializable value (no context_id, from + main gd) + - py_func: + context_id: ctx_other + expected_result: hello context + file: $(test_path)$(psep)py_func.py + func_name: get_context_value + key: $(test)_PASS + name: get serializable value (different context_id) + - py_func: + context_id: ctx_ns_test + expected_result: hello ns + file: $(test_path)$(psep)py_func.py + func_name: set_ns_value + key: $(test)_PASS + name: set non-serializable value + param: + - hello ns + - py_func: + context_id: ctx_ns_test + expected_result: hello ns + file: $(test_path)$(psep)py_func.py + func_name: get_ns_value + key: $(test)_PASS + name: get non-serializable value (same context_id) + filename: /home/renish/workspace/testium/code/test/validation/items/py_func/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/sleep.tum b/schema/test_schema/sleep.tum new file mode 100644 index 0000000..b6034b1 --- /dev/null +++ b/schema/test_schema/sleep.tum @@ -0,0 +1,87 @@ +config_file: +- param.yaml +- items/sleep/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: sleep test constants + values: + test: sleep + test_path: items/$(test) + - group: + name: sleep test + steps: + - sequence: + steps: + - sleep: + condition: $(validation_dialogs) + dialog: true + key: $(test)_PASS + name: Sleep timeout with dialogs + timeout: 3 + - sleep: + key: $(test)_PASS + name: Sleep timeout without dialog + timeout: 3.0 + - sleep: + key: $(test)_PASS + name: Sleep timeout in textual format + skipped: true + timeout: 1h 3m 2s + filename: /home/renish/workspace/testium/code/test/validation/items/sleep/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true diff --git a/schema/test_schema/unittest.tum b/schema/test_schema/unittest.tum new file mode 100644 index 0000000..141205a --- /dev/null +++ b/schema/test_schema/unittest.tum @@ -0,0 +1,84 @@ +config_file: +- param.yaml +- items/unittest/param.yaml +main: + name: Testium validation suite + steps: + - group: + name: Test preparation + steps: + - let: + condition: <| "$(os)" == "Linux" |> + name: Set test variables for Linux + values: + - terminal_prompt: $(linux_prompt) + - psep: / + - let: + condition: <| "$(os)" == "Windows" |> + name: Set test variables for Windows + values: + - terminal_prompt: $(windows_prompt) + - psep: \ + - group: + name: Group of tests + steps: + - let: + name: unittest test constants + values: + test: unittest + test_path: items/$(test) + - group: + name: unittest test + steps: + - sequence: + steps: + - unittest: + key: $(test)_PASS + name: Unittest item + test_file: /home/renish/workspace/testium/code/test/validation/items/unittest/unittest.py + test_method: test_01_pass + - unittest: + key: $(test)_FAIL + name: Unittest item + test_file: /home/renish/workspace/testium/code/test/validation/items/unittest/unittest.py + test_method: + - test_04_disabled + - test_03_fail + filename: /home/renish/workspace/testium/code/test/validation/items/unittest/test.tum + - sequence: + steps: + - report: + export: + - text: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.txt + - html: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.html + - junit: + key: $(test)_PASS + path: $(validation_report_path)$(psep)$(test)_PASS.junit + name: Expected PASS $(test) test + - report: + export: + - text: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.txt + - html: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.html + - junit: + key: $(test)_FAIL + path: $(validation_report_path)$(psep)$(test)_FAIL.junit + name: Expected FAIL $(test) test + filename: /home/renish/workspace/testium/code/test/validation/items/report.tum +report: + enabled: true + export: + junit: + file_name: $(validation_report_file).junit + path: $(validation_report_path) + sqlite: + file_name: $(validation_report_file).sqlite + path: $(validation_report_path) + log_stored: true