510 lines
16 KiB
TeX
510 lines
16 KiB
TeX
\section{Test items}
|
||
|
||
% ===== Tests items summary =====
|
||
\subsection{Tests items summary}
|
||
|
||
\begin{frame}{Test items - 1/2}
|
||
\small
|
||
\begin{NiceTabular}{l X}
|
||
\hline
|
||
\textbf{Item} & \textbf{Description} \\
|
||
\hline
|
||
\incode{check} & Checks for a value or expression \\
|
||
\incode{console} & Console steps (serial, terminal, telnet, tcp, ssh) \\
|
||
\incode{dialog\_choices} & Asks for a choice in list \\
|
||
\incode{dialog\_image} & Displays an image \\
|
||
\incode{dialog\_message} & Displays a message \\
|
||
\incode{dialog\_note} & Asks to enter a note \\
|
||
\incode{dialog\_question} & Asks a question with a yes/no choice \\
|
||
\incode{dialog\_references} & Asks for references \\
|
||
\incode{dialog\_value} & Asks for a value, entered manually \\
|
||
\incode{py\_func} & Python function call (from a file) \\
|
||
\incode{lua\_func} & Lua function call (from a file) \\
|
||
\hline
|
||
\end{NiceTabular}
|
||
\end{frame}
|
||
\begin{frame}{Test items - 2/2}
|
||
\small
|
||
\begin{NiceTabular}{l X}
|
||
\hline
|
||
\textbf{Item} & \textbf{Description} \\
|
||
\hline
|
||
\incode{group} & container for grouping things \\
|
||
\incode{jsonrpc} & JSON-RPC test item \\
|
||
\incode{let} & Defining variables \\
|
||
\incode{loop} & Container for repeating things \\
|
||
\incode{parallel} & Container for running branches concurrently \\
|
||
\incode{plot} & Runtime plot utility \\
|
||
\incode{report} & Extract a report file \\
|
||
\incode{run} & Runs a new instance of \tum{testium} \\
|
||
\incode{sleep} & Wait (with or without dialog) \\
|
||
\incode{unittest\_file} & Python unittest file \\
|
||
\hline
|
||
\end{NiceTabular}
|
||
\end{frame}
|
||
|
||
% ===== Tests items common features =====
|
||
\subsection{Tests items common features}
|
||
|
||
\begin{frame}{Items common attributes}
|
||
\begin{itemize}
|
||
\item Mandatory
|
||
\begin{itemize}
|
||
\item \incode{name} : The test item name
|
||
\end{itemize}
|
||
\item Optional
|
||
\begin{itemize}
|
||
\item \incode{stop\_on\_failure}: Stop the test if there is one failure
|
||
\item \incode{execute\_on\_stop} : Execution of the item when the test is stopped
|
||
\item \incode{skipped} : \incode{true} to skip the execution of the test
|
||
\item \incode{doc} : The documentation of the test item
|
||
\item \incode{key} : A Key used to filter the reports
|
||
\item \incode{condition} : If \incode{true}, will execute the test step. Otherwise, the test step is skipped
|
||
\item \incode{process\_result} : Processes the result
|
||
\item \incode{expected\_result} : Expected result of the item
|
||
\item \incode{store\_result} : Stores the result in a global variable
|
||
\item \incode{no\_fail} : If \incode{true}, forces the step result to be \pass{PASS}
|
||
\end{itemize}
|
||
\end{itemize}
|
||
\end{frame}
|
||
|
||
|
||
\begin{frame}{Items common variables}
|
||
\begin{itemize}
|
||
\item for all test items
|
||
\begin{itemize}
|
||
\item \incode{last\_step\_result} : contains the last test step result
|
||
\begin{itemize}
|
||
\item if no result returned by the test
|
||
\item \pass{PASS}, \fail{FAIL} or \incode{SKIPPED}
|
||
\end{itemize}
|
||
\item \incode{ts\_start\_<item\_name>} : The timestamp at the beginning of the item name
|
||
\item \incode{ts\_end\_<item\_name>} : The timestamp at the beginning of the item name
|
||
\end{itemize}
|
||
\item \incode{console} test item specific variables
|
||
\begin{itemize}
|
||
\item \incode{cn\_<item\_name>} : Containing the last data which has been read in the console
|
||
\end{itemize}
|
||
\item \incode{py\_func} test item specific variables
|
||
\begin{itemize}
|
||
\item \incode{pfn\_<item\_name>} : The returned value of the last \incode{py\_func} test item execution
|
||
\end{itemize}
|
||
\item \incode{lua\_func} test item specific variables
|
||
\begin{itemize}
|
||
\item \incode{lfn\_<item\_name>} : The returned value of the last \incode{lua\_func} test item execution
|
||
\end{itemize}
|
||
\item \incode{loop} test item specific variables
|
||
\begin{itemize}
|
||
\item \incode{loop\_index} : loop index (starting from 0)
|
||
\item \incode{loop\_index\_inverse} : loop index in reverse order
|
||
\item \incode{loop\_count} : total number of iterations
|
||
\item \incode{loop\_param} : current value of the loop iterator
|
||
\end{itemize}
|
||
\end{itemize}
|
||
\end{frame}
|
||
|
||
% ===== Tests items =====
|
||
\subsection{Tests items}
|
||
|
||
\begin{frame}[fragile]{\incode{console} test item}
|
||
\begin{columns}
|
||
\column{0.6\textwidth}
|
||
\begin{itemize}
|
||
\item \incode{open} : Open a console using the specified protocol
|
||
\begin{itemize}
|
||
\item \incode{telnet}
|
||
\item \incode{ssh}
|
||
\item \incode{serial}
|
||
\item \incode{rawtcp}
|
||
\item \incode{terminal}
|
||
\end{itemize}
|
||
\item \incode{close} : Close the console
|
||
\item \incode{write} : Write the string in the console
|
||
\item \incode{writeln} : Write the string in the console with \textbackslash{}n
|
||
\item \incode{read\_until} : Read until an expected string
|
||
\begin{itemize}
|
||
\item \incode{expected} : The expected string to be received
|
||
\item \incode{timeout} : The timeout to read the expected until it \fail{FAIL}s
|
||
\item \incode{no\_fail} : Do not \fail{FAIL} even if a timeout occurs
|
||
\end{itemize}
|
||
\end{itemize}
|
||
\column{0.4\textwidth}
|
||
\begin{listing}[H]
|
||
\begin{minted}{yaml}
|
||
- console:
|
||
name: test name in GUI
|
||
console_name: console name in dict
|
||
steps:
|
||
- open:
|
||
protocol: telnet
|
||
telnet_host: $(target_ip)
|
||
telnet_port: $(target_port)
|
||
- writeln: reset
|
||
- read_until: {expected: U-Boot, timeout: 50}
|
||
- write: $(boot_vxworks_1)
|
||
- writeln: $(boot_vxworks_2)
|
||
- read_until:
|
||
expected: U-Boot
|
||
timeout: 15
|
||
- read_until:
|
||
expected: Something that will never occurs
|
||
timeout: 5
|
||
no_fail: True
|
||
mute: True
|
||
- close:
|
||
\end{minted}
|
||
\caption{\incode{console} usage example}
|
||
\end{listing}
|
||
\end{columns}
|
||
\end{frame}
|
||
|
||
\begin{frame}[fragile]{\incode{py\_func} test item function call}
|
||
\begin{itemize}
|
||
\item The python file must import helper’s module \incode{py\_func.tm}
|
||
\item Calls the function \incode{exec} from a Python class
|
||
\begin{itemize}
|
||
\item The Python class must be inherited from \incode{tm.FunctionItem}
|
||
\begin{minted}[fontsize=\fontsize{6}{7}\selectfont]{python}
|
||
import py_func.tm as tm
|
||
|
||
class TestItemPyFunc(tm.FunctionItem)
|
||
|
||
def exec(param1, param2, param4, param4):
|
||
...
|
||
self.reportValue('my_reported_value', reported_value)
|
||
print(self.reportedValues())
|
||
return 10
|
||
\end{minted}
|
||
\end{itemize}
|
||
\item Calls a Python function by its name
|
||
\begin{minted}[fontsize=\fontsize{6}{7}\selectfont]{python}
|
||
def dummy_func(param1, param2, param4, param4):
|
||
...
|
||
return 10
|
||
\end{minted}
|
||
\begin{itemize}
|
||
\item To get a variable from the global dictionnary : \incode{tm.gd("global\_dict\_key")}
|
||
\item To set a variable from the global dictionnary : \incode{tm.setgd("global\_dict\_key", value)}
|
||
\end{itemize}
|
||
\item \incode{context\_id}: share a persistent subprocess across \incode{py\_func} items
|
||
\end{itemize}
|
||
\end{frame}
|
||
|
||
\begin{frame}[fragile]{\incode{lua\_func} test item function call}
|
||
\begin{itemize}
|
||
\item The lua file must import helper's module \incode{testium}
|
||
\item Calls a Lua function by its name
|
||
\begin{minted}[fontsize=\fontsize{6}{7}\selectfont]{lua}
|
||
tm = require("testium")
|
||
|
||
function dummy_func(param1, param2)
|
||
local value = tm.gd("global_dict_key")
|
||
...
|
||
return 10
|
||
end
|
||
\end{minted}
|
||
\begin{itemize}
|
||
\item To get a variable from the global dictionnary : \incode{tm.gd("global\_dict\_key")}
|
||
\item To set a variable from the global dictionnary : \incode{tm.setgd("global\_dict\_key", value)}
|
||
\end{itemize}
|
||
\item \incode{context\_id}: share a persistent subprocess across \incode{lua\_func} items
|
||
\end{itemize}
|
||
\end{frame}
|
||
|
||
\begin{frame}[fragile]{\incode{context\_id} — shared persistent subprocess}
|
||
\begin{columns}
|
||
\column{0.5\textwidth}
|
||
\begin{itemize}
|
||
\item By default, each \incode{py\_func}/\incode{lua\_func} call spawns its own subprocess
|
||
\begin{itemize}
|
||
\item module-level state is \textbf{not} preserved between calls
|
||
\end{itemize}
|
||
\item \incode{context\_id}: items sharing the same id reuse the \textbf{same} persistent subprocess
|
||
\begin{itemize}
|
||
\item subprocess lives until the end of the test run
|
||
\item required for non-JSON-serializable objects (connections, handles\ldots)
|
||
\end{itemize}
|
||
\item \incode{tm.setgd}/\incode{tm.gd} always work across all items for serializable values
|
||
\begin{itemize}
|
||
\item non-serializable values (\incode{py\_func} only) are kept locally in the subprocess
|
||
\end{itemize}
|
||
\end{itemize}
|
||
\column{0.5\textwidth}
|
||
\begin{minted}[fontsize=\fontsize{6}{7}\selectfont]{yaml}
|
||
- py_func:
|
||
name: open connection
|
||
file: my_script.py
|
||
func_name: open_connection
|
||
context_id: my_context
|
||
expected_result: ok
|
||
|
||
- py_func:
|
||
name: use connection
|
||
file: my_script.py
|
||
func_name: use_connection
|
||
context_id: my_context
|
||
expected_result: open
|
||
\end{minted}
|
||
\begin{minted}[fontsize=\fontsize{6}{7}\selectfont]{yaml}
|
||
- lua_func:
|
||
name: produce value
|
||
file: my_script.lua
|
||
func_name: produce
|
||
context_id: my_context
|
||
param:
|
||
- hello
|
||
|
||
- lua_func:
|
||
name: consume value
|
||
file: my_script.lua
|
||
func_name: consume
|
||
context_id: my_context
|
||
expected_result: hello
|
||
\end{minted}
|
||
\end{columns}
|
||
\end{frame}
|
||
|
||
\begin{frame}[fragile]{\incode{group} test item}
|
||
\begin{itemize}
|
||
\item \incode{steps}: The list of items in the group
|
||
\begin{minted}[fontsize=\fontsize{6}{7}\selectfont]{yaml}
|
||
- group:
|
||
name: Group Item
|
||
condition: "'$(OS)' == 'Linux'"
|
||
steps:
|
||
- unittest:
|
||
test_file: test_prod_rio6_8093.py
|
||
test_method:
|
||
...
|
||
- sleep:
|
||
timeout: 10
|
||
\end{minted}
|
||
\end{itemize}
|
||
\end{frame}
|
||
|
||
\begin{frame}[fragile]{\incode{loop} test item}
|
||
\begin{columns}
|
||
\column{0.6\textwidth}
|
||
\begin{itemize}
|
||
\item \incode{iterator}
|
||
\begin{itemize}
|
||
\item The number of iterations of the loop
|
||
\item The list of each iteration parameter
|
||
\item Not defined: infinite loop
|
||
\end{itemize}
|
||
\item \incode{steps}: The list of items to loop
|
||
\item \incode{exit\_condition}
|
||
\begin{itemize}
|
||
\item calls a Python function and stop the \incode{loop} if it does not return \incode{False}
|
||
\end{itemize}
|
||
\item Variables:
|
||
\begin{itemize}
|
||
\item \incode{loop\_param}: The loop iterator
|
||
\item \incode{loop\_index}: The loop index
|
||
\end{itemize}
|
||
\end{itemize}
|
||
\column{0.4\textwidth}
|
||
\begin{listing}[H]
|
||
\begin{minted}{yaml}
|
||
- loop:
|
||
name: Cycle Temperature
|
||
iterator: 10
|
||
steps:
|
||
- unittest:
|
||
test_file: test_prod_rio6_8093.py
|
||
- py_func:
|
||
name: function test item
|
||
file: scriptTestFile.py
|
||
func_name: funcToBeExecuted
|
||
param:
|
||
- $(loop_param)
|
||
exit_condition:
|
||
file: script_name.py
|
||
func_name: methodName
|
||
\end{minted}
|
||
\caption{\incode{loop} usage example}
|
||
\end{listing}
|
||
\end{columns}
|
||
\end{frame}
|
||
|
||
\begin{frame}[fragile]{\incode{parallel} test item}
|
||
\begin{columns}
|
||
\column{0.55\textwidth}
|
||
\begin{itemize}
|
||
\item \incode{branches}: list of concurrent branches; each branch has its own \incode{steps}
|
||
\item \incode{sync}
|
||
\begin{itemize}
|
||
\item \incode{all} (default): wait for every branch
|
||
\item \incode{any}: stop the others as soon as one finishes
|
||
\end{itemize}
|
||
\item \incode{wait\_for} (per branch): poll a condition before running the branch's steps
|
||
\item Each branch runs in its own thread:
|
||
\begin{itemize}
|
||
\item Live output prefixed \incode{[<branch>]}
|
||
\item One report row per branch (clean, no interleaving)
|
||
\end{itemize}
|
||
\end{itemize}
|
||
\column{0.45\textwidth}
|
||
\begin{listing}[H]
|
||
\begin{minted}[fontsize=\fontsize{6}{7}\selectfont]{yaml}
|
||
- parallel:
|
||
name: Read sensors
|
||
sync: all
|
||
branches:
|
||
- name: Temperature
|
||
steps:
|
||
- py_func:
|
||
name: read temp
|
||
file: sensors.py
|
||
func_name: read_temp
|
||
- name: Pressure
|
||
wait_for:
|
||
condition: <| "$(temp_ready)" == "1" |>
|
||
timeout: 5
|
||
steps:
|
||
- py_func:
|
||
name: read pressure
|
||
file: sensors.py
|
||
func_name: read_pressure
|
||
\end{minted}
|
||
\caption{\incode{parallel} usage example}
|
||
\end{listing}
|
||
\end{columns}
|
||
\end{frame}
|
||
|
||
\begin{frame}{\incode{dialog\_x} test items}
|
||
\begin{itemize}
|
||
\item Dialogs pop-up a message box with a question, information or more
|
||
\item Dialogs require an action of the operator
|
||
\item Available dialogs are:
|
||
\begin{itemize}
|
||
\item \incode{dialog\_message} : Display a message to the operator
|
||
\item \incode{dialog\_question} : Ask a question and \fail{FAIL} if the answer is no
|
||
\item \incode{dialog\_note} : The operator can write a small note
|
||
\item \incode{dialog\_value} : The operator enter a value
|
||
\item \incode{dialog\_references}: Ask for the reference, revision and serial number
|
||
\item \incode{dialog\_image} : A \incode{dialog\_question} with an image
|
||
\item \incode{dialog\_choices} : A choice of items to check
|
||
\end{itemize}
|
||
\end{itemize}
|
||
\end{frame}
|
||
|
||
\begin{frame}[fragile]{\incode{let} test item}
|
||
\begin{itemize}
|
||
\item Add or set a variable in the global dictionary
|
||
\begin{itemize}
|
||
\item \incode{values}: list of key/values which are to be recorded
|
||
\end{itemize}
|
||
\begin{minted}[fontsize=\fontsize{6}{7}\selectfont]{yaml}
|
||
- let:
|
||
name: Let Item
|
||
values:
|
||
key1: value1
|
||
key2: value2
|
||
\end{minted}
|
||
\end{itemize}
|
||
\end{frame}
|
||
|
||
\begin{frame}[fragile]{\incode{check} test item}
|
||
\begin{itemize}
|
||
\item A simple test item returning \fail{FAIL} if one of its \incode{values} is \incode{false}
|
||
\begin{minted}[fontsize=\fontsize{6}{7}\selectfont]{yaml}
|
||
- check:
|
||
name: check test item example
|
||
values:
|
||
- True
|
||
- <|$(last_step_result) == 3|>
|
||
- <|"my string" in "$(my_global_variable)"|>
|
||
\end{minted}
|
||
\end{itemize}
|
||
\end{frame}
|
||
|
||
\begin{frame}[fragile]{\incode{sleep} test item}
|
||
\begin{itemize}
|
||
\item sleeps the requested time
|
||
\begin{itemize}
|
||
\item \incode{timeout}: the time to sleep in seconds
|
||
\item \incode{dialog}: if the sleep shows a dialog (\incode{false} by default)
|
||
\end{itemize}
|
||
\begin{minted}[fontsize=\fontsize{6}{7}\selectfont]{yaml}
|
||
- sleep:
|
||
name: sleeps 10 secs
|
||
timeout: 10
|
||
dialog: true
|
||
\end{minted}
|
||
\end{itemize}
|
||
\end{frame}
|
||
|
||
\begin{frame}[fragile]{\incode{json\_rpc} test item}
|
||
\begin{columns}
|
||
\column{0.6\textwidth}
|
||
\begin{itemize}
|
||
\item JSON-RPC is a remote procedure call standard
|
||
\item transport protocol can be chosen among:
|
||
\begin{itemize}
|
||
\item udp
|
||
\item a previously opened \incode{console}
|
||
\end{itemize}
|
||
\end{itemize}
|
||
\begin{listing}[H]
|
||
\begin{minted}{yaml}
|
||
- json_rpc:
|
||
name: JSONRPC UDP query waiting for reception
|
||
key: $(test)_PASS
|
||
udp: {server: localhost, snd_port: 4323, rcv_port: 8765}
|
||
timeout: 1
|
||
steps:
|
||
- open:
|
||
- query:
|
||
name: echo
|
||
method: echo
|
||
params:
|
||
- Hello World
|
||
- {a: 1, b: "hello"}
|
||
timeout: 1
|
||
- close:
|
||
\end{minted}
|
||
\caption{\incode{json\_rpc} query example, waiting for server answer}
|
||
\end{listing}
|
||
\column{0.4\textwidth}
|
||
\begin{listing}[H]
|
||
\begin{minted}{yaml}
|
||
- json_rpc:
|
||
name: JSONRPC UDP query not waiting (only send)
|
||
key: $(test)_PASS
|
||
udp: {server: localhost, snd_port: 4323, rcv_port: 8765}
|
||
timeout: 1
|
||
steps:
|
||
- open:
|
||
- query:
|
||
method: echo
|
||
params:
|
||
- {b: "olleh", a: -1}
|
||
- World Hello
|
||
id: 3095372
|
||
no_wait: True
|
||
|
||
[...]
|
||
- json_rpc:
|
||
name: JSONRPC UDP Reception
|
||
key: $(test)_PASS
|
||
udp: {server: localhost, snd_port: 4323, rcv_port: 8765}
|
||
timeout: 1
|
||
steps:
|
||
- receive:
|
||
id: 3095372
|
||
- close:
|
||
\end{minted}
|
||
\caption{\incode{json\_rpc} query and answer separated}
|
||
\end{listing}
|
||
\end{columns}
|
||
\end{frame}
|
||
|
||
\begin{frame}{Other test items}
|
||
\begin{itemize}
|
||
\item \incode{plot}: Displays and stores time series data
|
||
\item \incode{run}: Execute a new instance of \tum{testium}
|
||
\item \incode{unittest\_file}: Runs a Python file based on the unittest framework
|
||
\end{itemize}
|
||
|
||
\end{frame} |