diff options
Diffstat (limited to 'tools/testing/selftests/tc-testing')
13 files changed, 615 insertions, 144 deletions
diff --git a/tools/testing/selftests/tc-testing/.gitignore b/tools/testing/selftests/tc-testing/.gitignore index 7a60b85e148f..c26d72e0166f 100644 --- a/tools/testing/selftests/tc-testing/.gitignore +++ b/tools/testing/selftests/tc-testing/.gitignore @@ -1,2 +1,6 @@ __pycache__/ *.pyc +plugins/ +*.xml +*.tap +tdc_config_local.py diff --git a/tools/testing/selftests/tc-testing/TdcPlugin.py b/tools/testing/selftests/tc-testing/TdcPlugin.py index 3ee9a6dacb52..b980a565fa89 100644 --- a/tools/testing/selftests/tc-testing/TdcPlugin.py +++ b/tools/testing/selftests/tc-testing/TdcPlugin.py @@ -18,12 +18,13 @@ class TdcPlugin: if self.args.verbose > 1: print(' -- {}.post_suite'.format(self.sub_class)) - def pre_case(self, test_ordinal, testid): + def pre_case(self, testid, test_name, test_skip): '''run commands before test_runner does one test''' if self.args.verbose > 1: print(' -- {}.pre_case'.format(self.sub_class)) self.args.testid = testid - self.args.test_ordinal = test_ordinal + self.args.test_name = test_name + self.args.test_skip = test_skip def post_case(self): '''run commands after test_runner does one test''' diff --git a/tools/testing/selftests/tc-testing/TdcResults.py b/tools/testing/selftests/tc-testing/TdcResults.py new file mode 100644 index 000000000000..1e4d95fdf8d0 --- /dev/null +++ b/tools/testing/selftests/tc-testing/TdcResults.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +from enum import Enum + +class ResultState(Enum): + noresult = -1 + skip = 0 + success = 1 + fail = 2 + +class TestResult: + def __init__(self, test_id="", test_name=""): + self.test_id = test_id + self.test_name = test_name + self.result = ResultState.noresult + self.failmsg = "" + self.errormsg = "" + self.steps = [] + + def set_result(self, result): + if (isinstance(result, ResultState)): + self.result = result + return True + else: + raise TypeError('Unknown result type, must be type ResultState') + + def get_result(self): + return self.result + + def set_errormsg(self, errormsg): + self.errormsg = errormsg + return True + + def append_errormsg(self, errormsg): + self.errormsg = '{}\n{}'.format(self.errormsg, errormsg) + + def get_errormsg(self): + return self.errormsg + + def set_failmsg(self, failmsg): + self.failmsg = failmsg + return True + + def append_failmsg(self, failmsg): + self.failmsg = '{}\n{}'.format(self.failmsg, failmsg) + + def get_failmsg(self): + return self.failmsg + + def add_steps(self, newstep): + if type(newstep) == list: + self.steps.extend(newstep) + elif type(newstep) == str: + self.steps.append(step) + else: + raise TypeError('TdcResults.add_steps() requires a list or str') + + def get_executed_steps(self): + return self.steps + +class TestSuiteReport(): + _testsuite = [] + + def add_resultdata(self, result_data): + if isinstance(result_data, TestResult): + self._testsuite.append(result_data) + return True + + def count_tests(self): + return len(self._testsuite) + + def count_failures(self): + return sum(1 for t in self._testsuite if t.result == ResultState.fail) + + def count_skips(self): + return sum(1 for t in self._testsuite if t.result == ResultState.skip) + + def find_result(self, test_id): + return next((tr for tr in self._testsuite if tr.test_id == test_id), None) + + def update_result(self, result_data): + orig = self.find_result(result_data.test_id) + if orig != None: + idx = self._testsuite.index(orig) + self._testsuite[idx] = result_data + else: + self.add_resultdata(result_data) + + def format_tap(self): + ftap = "" + ftap += '1..{}\n'.format(self.count_tests()) + index = 1 + for t in self._testsuite: + if t.result == ResultState.fail: + ftap += 'not ' + ftap += 'ok {} {} - {}'.format(str(index), t.test_id, t.test_name) + if t.result == ResultState.skip or t.result == ResultState.noresult: + ftap += ' # skipped - {}\n'.format(t.errormsg) + elif t.result == ResultState.fail: + if len(t.steps) > 0: + ftap += '\tCommands executed in this test case:' + for step in t.steps: + ftap += '\n\t\t{}'.format(step) + ftap += '\n\t{}'.format(t.failmsg) + ftap += '\n' + index += 1 + return ftap + + def format_xunit(self): + from xml.sax.saxutils import escape + xunit = "<testsuites>\n" + xunit += '\t<testsuite tests=\"{}\" skips=\"{}\">\n'.format(self.count_tests(), self.count_skips()) + for t in self._testsuite: + xunit += '\t\t<testcase classname=\"{}\" '.format(escape(t.test_id)) + xunit += 'name=\"{}\">\n'.format(escape(t.test_name)) + if t.failmsg: + xunit += '\t\t\t<failure>\n' + if len(t.steps) > 0: + xunit += 'Commands executed in this test case:\n' + for step in t.steps: + xunit += '\t{}\n'.format(escape(step)) + xunit += 'FAILURE: {}\n'.format(escape(t.failmsg)) + xunit += '\t\t\t</failure>\n' + if t.errormsg: + xunit += '\t\t\t<error>\n{}\n'.format(escape(t.errormsg)) + xunit += '\t\t\t</error>\n' + if t.result == ResultState.skip: + xunit += '\t\t\t<skipped/>\n' + xunit += '\t\t</testcase>\n' + xunit += '\t</testsuite>\n' + xunit += '</testsuites>\n' + return xunit diff --git a/tools/testing/selftests/tc-testing/bpf/Makefile b/tools/testing/selftests/tc-testing/bpf/Makefile index dc92eb271d9a..be5a5e542804 100644 --- a/tools/testing/selftests/tc-testing/bpf/Makefile +++ b/tools/testing/selftests/tc-testing/bpf/Makefile @@ -4,6 +4,7 @@ APIDIR := ../../../../include/uapi TEST_GEN_FILES = action.o top_srcdir = ../../../../.. +KSFT_KHDR_INSTALL := 1 include ../../lib.mk CLANG ?= clang diff --git a/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt b/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt index 17b267dedbd9..a28571aff0e1 100644 --- a/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt +++ b/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt @@ -33,6 +33,11 @@ Each test case has required data: id: A unique alphanumeric value to identify a particular test case name: Descriptive name that explains the command under test +skip: A completely optional key, if the corresponding value is "yes" + then tdc will not execute the test case in question. However, + this test case will still appear in the results output but + marked as skipped. This key can be placed anywhere inside the + test case at the top level. category: A list of single-word descriptions covering what the command under test is testing. Example: filter, actions, u32, gact, etc. setup: The list of commands required to ensure the command under test diff --git a/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py b/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py index 477a7bd7d7fb..4bb866575ea1 100644 --- a/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py +++ b/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py @@ -11,6 +11,7 @@ from string import Template import subprocess import time from TdcPlugin import TdcPlugin +from TdcResults import * from tdc_config import * @@ -21,6 +22,7 @@ class SubPlugin(TdcPlugin): def __init__(self): self.sub_class = 'valgrind/SubPlugin' self.tap = '' + self._tsr = TestSuiteReport() super().__init__() def pre_suite(self, testcount, testidlist): @@ -34,10 +36,14 @@ class SubPlugin(TdcPlugin): def post_suite(self, index): '''run commands after test_runner goes into a test loop''' super().post_suite(index) - self._add_to_tap('\n|---\n') if self.args.verbose > 1: print('{}.post_suite'.format(self.sub_class)) - print('{}'.format(self.tap)) + #print('{}'.format(self.tap)) + for xx in range(index - 1, self.testcount): + res = TestResult('{}-mem'.format(self.testidlist[xx]), 'Test skipped') + res.set_result(ResultState.skip) + res.set_errormsg('Skipped because of prior setup/teardown failure') + self._add_results(res) if self.args.verbose < 4: subprocess.check_output('rm -f vgnd-*.log', shell=True) @@ -96,6 +102,14 @@ class SubPlugin(TdcPlugin): if not self.args.valgrind: return + res = TestResult('{}-mem'.format(self.args.testid), + '{} memory leak check'.format(self.args.test_name)) + if self.args.test_skip: + res.set_result(ResultState.skip) + res.set_errormsg('Test case designated as skipped.') + self._add_results(res) + return + self.definitely_lost_re = re.compile( r'definitely lost:\s+([,0-9]+)\s+bytes in\s+([,0-9]+)\sblocks', re.MULTILINE | re.DOTALL) self.indirectly_lost_re = re.compile( @@ -130,13 +144,17 @@ class SubPlugin(TdcPlugin): mem_results = '' if (def_num > 0) or (ind_num > 0) or (pos_num > 0) or (nle_num > 0): mem_results += 'not ' + res.set_result(ResultState.fail) + res.set_failmsg('Memory leak detected') + res.append_failmsg(content) + else: + res.set_result(ResultState.success) + + self._add_results(res) + - mem_results += 'ok {} - {}-mem # {}\n'.format( - self.args.test_ordinal, self.args.testid, 'memory leak check') - self._add_to_tap(mem_results) - if mem_results.startswith('not '): - print('{}'.format(content)) - self._add_to_tap(content) + def _add_results(self, res): + self._tsr.add_resultdata(res) def _add_to_tap(self, more_tap_output): self.tap += more_tap_output diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/ife.json b/tools/testing/selftests/tc-testing/tc-tests/actions/ife.json index 637ea0219617..0da3545cabdb 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/actions/ife.json +++ b/tools/testing/selftests/tc-testing/tc-tests/actions/ife.json @@ -17,7 +17,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow mark pass index 2", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 2", - "matchPattern": "action order [0-9]*: ife encode action pass.*type 0xED3E.*allow mark.*index 2", + "matchPattern": "action order [0-9]*: ife encode action pass.*type 0[xX]ED3E.*allow mark.*index 2", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -41,7 +41,7 @@ "cmdUnderTest": "$TC actions add action ife encode use mark 10 pipe index 2", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 2", - "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0xED3E.*use mark.*index 2", + "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0[xX]ED3E.*use mark.*index 2", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -65,7 +65,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow mark continue index 2", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 2", - "matchPattern": "action order [0-9]*: ife encode action continue.*type 0xED3E.*allow mark.*index 2", + "matchPattern": "action order [0-9]*: ife encode action continue.*type 0[xX]ED3E.*allow mark.*index 2", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -89,7 +89,7 @@ "cmdUnderTest": "$TC actions add action ife encode use mark 789 drop index 2", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 2", - "matchPattern": "action order [0-9]*: ife encode action drop.*type 0xED3E.*use mark 789.*index 2", + "matchPattern": "action order [0-9]*: ife encode action drop.*type 0[xX]ED3E.*use mark 789.*index 2", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -113,7 +113,7 @@ "cmdUnderTest": "$TC actions add action ife encode use mark 656768 reclassify index 2", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 2", - "matchPattern": "action order [0-9]*: ife encode action reclassify.*type 0xED3E.*use mark 656768.*index 2", + "matchPattern": "action order [0-9]*: ife encode action reclassify.*type 0[xX]ED3E.*use mark 656768.*index 2", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -137,7 +137,7 @@ "cmdUnderTest": "$TC actions add action ife encode use mark 65 jump 1 index 2", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 2", - "matchPattern": "action order [0-9]*: ife encode action jump 1.*type 0xED3E.*use mark 65.*index 2", + "matchPattern": "action order [0-9]*: ife encode action jump 1.*type 0[xX]ED3E.*use mark 65.*index 2", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -161,7 +161,7 @@ "cmdUnderTest": "$TC actions add action ife encode use mark 4294967295 reclassify index 90", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 90", - "matchPattern": "action order [0-9]*: ife encode action reclassify.*type 0xED3E.*use mark 4294967295.*index 90", + "matchPattern": "action order [0-9]*: ife encode action reclassify.*type 0[xX]ED3E.*use mark 4294967295.*index 90", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -185,7 +185,7 @@ "cmdUnderTest": "$TC actions add action ife encode use mark 4294967295999 pipe index 90", "expExitCode": "255", "verifyCmd": "$TC actions get action ife index 90", - "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0xED3E.*use mark 4294967295999.*index 90", + "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0[xX]ED3E.*use mark 4294967295999.*index 90", "matchCount": "0", "teardown": [] }, @@ -207,7 +207,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow prio pass index 9", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 9", - "matchPattern": "action order [0-9]*: ife encode action pass.*type 0xED3E.*allow prio.*index 9", + "matchPattern": "action order [0-9]*: ife encode action pass.*type 0[xX]ED3E.*allow prio.*index 9", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -231,7 +231,7 @@ "cmdUnderTest": "$TC actions add action ife encode use prio 7 pipe index 9", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 9", - "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0xED3E.*use prio 7.*index 9", + "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0[xX]ED3E.*use prio 7.*index 9", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -255,7 +255,7 @@ "cmdUnderTest": "$TC actions add action ife encode use prio 3 continue index 9", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 9", - "matchPattern": "action order [0-9]*: ife encode action continue.*type 0xED3E.*use prio 3.*index 9", + "matchPattern": "action order [0-9]*: ife encode action continue.*type 0[xX]ED3E.*use prio 3.*index 9", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -279,7 +279,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow prio drop index 9", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 9", - "matchPattern": "action order [0-9]*: ife encode action drop.*type 0xED3E.*allow prio.*index 9", + "matchPattern": "action order [0-9]*: ife encode action drop.*type 0[xX]ED3E.*allow prio.*index 9", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -303,7 +303,7 @@ "cmdUnderTest": "$TC actions add action ife encode use prio 998877 reclassify index 9", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 9", - "matchPattern": "action order [0-9]*: ife encode action reclassify.*type 0xED3E.*use prio 998877.*index 9", + "matchPattern": "action order [0-9]*: ife encode action reclassify.*type 0[xX]ED3E.*use prio 998877.*index 9", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -327,7 +327,7 @@ "cmdUnderTest": "$TC actions add action ife encode use prio 998877 jump 10 index 9", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 9", - "matchPattern": "action order [0-9]*: ife encode action jump 10.*type 0xED3E.*use prio 998877.*index 9", + "matchPattern": "action order [0-9]*: ife encode action jump 10.*type 0[xX]ED3E.*use prio 998877.*index 9", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -351,7 +351,7 @@ "cmdUnderTest": "$TC actions add action ife encode use prio 4294967295 reclassify index 99", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 99", - "matchPattern": "action order [0-9]*: ife encode action reclassify.*type 0xED3E.*use prio 4294967295.*index 99", + "matchPattern": "action order [0-9]*: ife encode action reclassify.*type 0[xX]ED3E.*use prio 4294967295.*index 99", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -375,7 +375,7 @@ "cmdUnderTest": "$TC actions add action ife encode use prio 4294967298 pipe index 99", "expExitCode": "255", "verifyCmd": "$TC actions get action ife index 99", - "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0xED3E.*use prio 4294967298.*index 99", + "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0[xX]ED3E.*use prio 4294967298.*index 99", "matchCount": "0", "teardown": [] }, @@ -397,7 +397,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow tcindex pass index 1", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 1", - "matchPattern": "action order [0-9]*: ife encode action pass.*type 0xED3E.*allow tcindex.*index 1", + "matchPattern": "action order [0-9]*: ife encode action pass.*type 0[xX]ED3E.*allow tcindex.*index 1", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -421,7 +421,7 @@ "cmdUnderTest": "$TC actions add action ife encode use tcindex 111 pipe index 1", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 1", - "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0xED3E.*use tcindex 111.*index 1", + "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0[xX]ED3E.*use tcindex 111.*index 1", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -445,7 +445,7 @@ "cmdUnderTest": "$TC actions add action ife encode use tcindex 1 continue index 1", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 1", - "matchPattern": "action order [0-9]*: ife encode action continue.*type 0xED3E.*use tcindex 1.*index 1", + "matchPattern": "action order [0-9]*: ife encode action continue.*type 0[xX]ED3E.*use tcindex 1.*index 1", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -469,7 +469,7 @@ "cmdUnderTest": "$TC actions add action ife encode use tcindex 1 continue index 1", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 1", - "matchPattern": "action order [0-9]*: ife encode action continue.*type 0xED3E.*use tcindex 1.*index 1", + "matchPattern": "action order [0-9]*: ife encode action continue.*type 0[xX]ED3E.*use tcindex 1.*index 1", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -493,7 +493,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow tcindex drop index 77", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 77", - "matchPattern": "action order [0-9]*: ife encode action drop.*type 0xED3E.*allow tcindex.*index 77", + "matchPattern": "action order [0-9]*: ife encode action drop.*type 0[xX]ED3E.*allow tcindex.*index 77", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -517,7 +517,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow tcindex reclassify index 77", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 77", - "matchPattern": "action order [0-9]*: ife encode action reclassify.*type 0xED3E.*allow tcindex.*index 77", + "matchPattern": "action order [0-9]*: ife encode action reclassify.*type 0[xX]ED3E.*allow tcindex.*index 77", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -541,7 +541,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow tcindex jump 999 index 77", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 77", - "matchPattern": "action order [0-9]*: ife encode action jump 999.*type 0xED3E.*allow tcindex.*index 77", + "matchPattern": "action order [0-9]*: ife encode action jump 999.*type 0[xX]ED3E.*allow tcindex.*index 77", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -565,7 +565,7 @@ "cmdUnderTest": "$TC actions add action ife encode use tcindex 65535 pass index 1", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 1", - "matchPattern": "action order [0-9]*: ife encode action pass.*type 0xED3E.*use tcindex 65535.*index 1", + "matchPattern": "action order [0-9]*: ife encode action pass.*type 0[xX]ED3E.*use tcindex 65535.*index 1", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -589,7 +589,7 @@ "cmdUnderTest": "$TC actions add action ife encode use tcindex 65539 pipe index 1", "expExitCode": "255", "verifyCmd": "$TC actions get action ife index 1", - "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0xED3E.*use tcindex 65539.*index 1", + "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0[xX]ED3E.*use tcindex 65539.*index 1", "matchCount": "0", "teardown": [] }, @@ -611,7 +611,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow mark src 00:11:22:33:44:55 pipe index 1", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 1", - "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0xED3E.*allow mark src 00:11:22:33:44:55.*index 1", + "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0[xX]ED3E.*allow mark src 00:11:22:33:44:55.*index 1", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -635,7 +635,7 @@ "cmdUnderTest": "$TC actions add action ife encode use prio 9876 dst 00:11:22:33:44:55 reclassify index 1", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 1", - "matchPattern": "action order [0-9]*: ife encode action reclassify.*type 0xED3E.*use prio 9876 dst 00:11:22:33:44:55.*index 1", + "matchPattern": "action order [0-9]*: ife encode action reclassify.*type 0[xX]ED3E.*use prio 9876 dst 00:11:22:33:44:55.*index 1", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -659,7 +659,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow tcindex src 00:aa:bb:cc:dd:ee dst 00:11:22:33:44:55 pass index 11", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 11", - "matchPattern": "action order [0-9]*: ife encode action pass.*type 0xED3E.*allow tcindex dst 00:11:22:33:44:55 src 00:aa:bb:cc:dd:ee .*index 11", + "matchPattern": "action order [0-9]*: ife encode action pass.*type 0[xX]ED3E.*allow tcindex dst 00:11:22:33:44:55 src 00:aa:bb:cc:dd:ee .*index 11", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -683,7 +683,7 @@ "cmdUnderTest": "$TC actions add action ife encode use mark 7 type 0xfefe pass index 1", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 1", - "matchPattern": "action order [0-9]*: ife encode action pass.*type 0xFEFE.*use mark 7.*index 1", + "matchPattern": "action order [0-9]*: ife encode action pass.*type 0[xX]FEFE.*use mark 7.*index 1", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -707,7 +707,7 @@ "cmdUnderTest": "$TC actions add action ife encode use prio 444 type 0xabba pipe index 21", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 21", - "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0xABBA.*use prio 444.*index 21", + "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0[xX]ABBA.*use prio 444.*index 21", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -731,7 +731,7 @@ "cmdUnderTest": "$TC actions add action ife encode use tcindex 5000 type 0xabcd reclassify index 21", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 21", - "matchPattern": "action order [0-9]*: ife encode action reclassify.*type 0xABCD.*use tcindex 5000.*index 21", + "matchPattern": "action order [0-9]*: ife encode action reclassify.*type 0[xX]ABCD.*use tcindex 5000.*index 21", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -739,7 +739,7 @@ }, { "id": "fac3", - "name": "Create valid ife encode action with index at 32-bit maximnum", + "name": "Create valid ife encode action with index at 32-bit maximum", "category": [ "actions", "ife" @@ -755,7 +755,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow mark pass index 4294967295", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 4294967295", - "matchPattern": "action order [0-9]*: ife encode action pass.*type 0xED3E.*allow mark.*index 4294967295", + "matchPattern": "action order [0-9]*: ife encode action pass.*type 0[xX]ED3E.*allow mark.*index 4294967295", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -779,7 +779,7 @@ "cmdUnderTest": "$TC actions add action ife decode pass index 1", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 1", - "matchPattern": "action order [0-9]*: ife decode action pass.*type 0x0.*allow mark allow tcindex allow prio.*index 1", + "matchPattern": "action order [0-9]*: ife decode action pass.*type 0(x0)?.*allow mark allow tcindex allow prio.*index 1", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -803,7 +803,7 @@ "cmdUnderTest": "$TC actions add action ife decode pipe index 1", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 1", - "matchPattern": "action order [0-9]*: ife decode action pipe.*type 0x0.*allow mark allow tcindex allow prio.*index 1", + "matchPattern": "action order [0-9]*: ife decode action pipe.*type 0(x0)?.*allow mark allow tcindex allow prio.*index 1", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -827,7 +827,7 @@ "cmdUnderTest": "$TC actions add action ife decode continue index 1", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 1", - "matchPattern": "action order [0-9]*: ife decode action continue.*type 0x0.*allow mark allow tcindex allow prio.*index 1", + "matchPattern": "action order [0-9]*: ife decode action continue.*type 0(x0)?.*allow mark allow tcindex allow prio.*index 1", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -851,7 +851,7 @@ "cmdUnderTest": "$TC actions add action ife decode drop index 1", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 1", - "matchPattern": "action order [0-9]*: ife decode action drop.*type 0x0.*allow mark allow tcindex allow prio.*index 1", + "matchPattern": "action order [0-9]*: ife decode action drop.*type 0(x0)?.*allow mark allow tcindex allow prio.*index 1", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -875,7 +875,7 @@ "cmdUnderTest": "$TC actions add action ife decode reclassify index 1", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 1", - "matchPattern": "action order [0-9]*: ife decode action reclassify.*type 0x0.*allow mark allow tcindex allow prio.*index 1", + "matchPattern": "action order [0-9]*: ife decode action reclassify.*type 0(x0)?.*allow mark allow tcindex allow prio.*index 1", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -899,7 +899,7 @@ "cmdUnderTest": "$TC actions add action ife decode jump 10 index 1", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 1", - "matchPattern": "action order [0-9]*: ife decode action jump 10.*type 0x0.*allow mark allow tcindex allow prio.*index 1", + "matchPattern": "action order [0-9]*: ife decode action jump 10.*type 0(x0)?.*allow mark allow tcindex allow prio.*index 1", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -923,7 +923,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow mark pass index 4294967295999", "expExitCode": "255", "verifyCmd": "$TC actions get action ife index 4294967295999", - "matchPattern": "action order [0-9]*: ife encode action pass.*type 0xED3E.*allow mark.*index 4294967295999", + "matchPattern": "action order [0-9]*: ife encode action pass.*type 0[xX]ED3E.*allow mark.*index 4294967295999", "matchCount": "0", "teardown": [] }, @@ -945,7 +945,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow mark kuka index 4", "expExitCode": "255", "verifyCmd": "$TC actions get action ife index 4", - "matchPattern": "action order [0-9]*: ife encode action kuka.*type 0xED3E.*allow mark.*index 4", + "matchPattern": "action order [0-9]*: ife encode action kuka.*type 0[xX]ED3E.*allow mark.*index 4", "matchCount": "0", "teardown": [] }, @@ -967,7 +967,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow prio pipe index 4 cookie aabbccddeeff112233445566778800a1", "expExitCode": "0", "verifyCmd": "$TC actions get action ife index 4", - "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0xED3E.*allow prio.*index 4.*cookie aabbccddeeff112233445566778800a1", + "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0[xX]ED3E.*allow prio.*index 4.*cookie aabbccddeeff112233445566778800a1", "matchCount": "1", "teardown": [ "$TC actions flush action ife" @@ -991,7 +991,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow foo pipe index 4", "expExitCode": "255", "verifyCmd": "$TC actions get action ife index 4", - "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0xED3E.*allow foo.*index 4", + "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0[xX]ED3E.*allow foo.*index 4", "matchCount": "0", "teardown": [] }, @@ -1013,7 +1013,7 @@ "cmdUnderTest": "$TC actions add action ife encode allow prio type 70000 pipe index 4", "expExitCode": "255", "verifyCmd": "$TC actions get action ife index 4", - "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0x11170.*allow prio.*index 4", + "matchPattern": "action order [0-9]*: ife encode action pipe.*type 0[xX]11170.*allow prio.*index 4", "matchCount": "0", "teardown": [] }, diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/tunnel_key.json b/tools/testing/selftests/tc-testing/tc-tests/actions/tunnel_key.json index 10b2d894e436..e7e15a7336b6 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/actions/tunnel_key.json +++ b/tools/testing/selftests/tc-testing/tc-tests/actions/tunnel_key.json @@ -82,35 +82,6 @@ ] }, { - "id": "ba4e", - "name": "Add tunnel_key set action with missing mandatory id parameter", - "category": [ - "actions", - "tunnel_key" - ], - "setup": [ - [ - "$TC actions flush action tunnel_key", - 0, - 1, - 255 - ] - ], - "cmdUnderTest": "$TC actions add action tunnel_key set src_ip 10.10.10.1 dst_ip 20.20.20.2", - "expExitCode": "255", - "verifyCmd": "$TC actions list action tunnel_key", - "matchPattern": "action order [0-9]+: tunnel_key set.*src_ip 10.10.10.1.*dst_ip 20.20.20.2", - "matchCount": "0", - "teardown": [ - [ - "$TC actions flush action tunnel_key", - 0, - 1, - 255 - ] - ] - }, - { "id": "a5e0", "name": "Add tunnel_key set action with invalid src_ip parameter", "category": [ @@ -634,7 +605,7 @@ "cmdUnderTest": "$TC actions add action tunnel_key set src_ip 10.10.10.1 dst_ip 10.10.10.2 id 7 index 4 cookie aa11bb22cc33dd44ee55ff66aa11b1b2", "expExitCode": "0", "verifyCmd": "$TC actions get action tunnel_key index 4", - "matchPattern": "action order [0-9]+: tunnel_key.*set.*src_ip 10.10.10.1.*dst_ip 10.10.10.2.*key_id 7.*dst_port 0.*csum pipe.*index 4 ref.*cookie aa11bb22cc33dd44ee55ff66aa11b1b2", + "matchPattern": "action order [0-9]+: tunnel_key.*set.*src_ip 10.10.10.1.*dst_ip 10.10.10.2.*key_id 7.*csum pipe.*index 4 ref.*cookie aa11bb22cc33dd44ee55ff66aa11b1b2", "matchCount": "1", "teardown": [ "$TC actions flush action tunnel_key" diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/concurrency.json b/tools/testing/selftests/tc-testing/tc-tests/filters/concurrency.json new file mode 100644 index 000000000000..9002714b1851 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/filters/concurrency.json @@ -0,0 +1,177 @@ +[ + { + "id": "e41d", + "name": "Add 1M flower filters with 10 parallel tc instances", + "category": [ + "filter", + "flower", + "concurrency" + ], + "setup": [ + "/bin/mkdir $BATCH_DIR", + "$TC qdisc add dev $DEV2 ingress", + "./tdc_multibatch.py $DEV2 $BATCH_DIR 100000 10 add" + ], + "cmdUnderTest": "find $BATCH_DIR/add* -print | xargs -n 1 -P 10 $TC -b", + "expExitCode": "0", + "verifyCmd": "$TC -s filter show dev $DEV2 ingress", + "matchPattern": "filter protocol ip pref 1 flower chain 0 handle", + "matchCount": "1000000", + "teardown": [ + "$TC qdisc del dev $DEV2 ingress", + "/bin/rm -rf $BATCH_DIR" + ] + }, + { + "id": "6f52", + "name": "Delete 1M flower filters with 10 parallel tc instances", + "category": [ + "filter", + "flower", + "concurrency" + ], + "setup": [ + "/bin/mkdir $BATCH_DIR", + "$TC qdisc add dev $DEV2 ingress", + "./tdc_multibatch.py $DEV2 $BATCH_DIR 1000000 1 add", + "$TC -b $BATCH_DIR/add_0", + "./tdc_multibatch.py $DEV2 $BATCH_DIR 100000 10 del" + ], + "cmdUnderTest": "find $BATCH_DIR/del* -print | xargs -n 1 -P 10 $TC -b", + "expExitCode": "0", + "verifyCmd": "$TC -s filter show dev $DEV2 ingress", + "matchPattern": "filter protocol ip pref 1 flower chain 0 handle", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV2 ingress", + "/bin/rm -rf $BATCH_DIR" + ] + }, + { + "id": "c9da", + "name": "Replace 1M flower filters with 10 parallel tc instances", + "category": [ + "filter", + "flower", + "concurrency" + ], + "setup": [ + "/bin/mkdir $BATCH_DIR", + "$TC qdisc add dev $DEV2 ingress", + "./tdc_multibatch.py $DEV2 $BATCH_DIR 1000000 1 add", + "$TC -b $BATCH_DIR/add_0", + "./tdc_multibatch.py $DEV2 $BATCH_DIR 100000 10 replace" + ], + "cmdUnderTest": "find $BATCH_DIR/replace* -print | xargs -n 1 -P 10 $TC -b", + "expExitCode": "0", + "verifyCmd": "$TC -s filter show dev $DEV2 ingress", + "matchPattern": "filter protocol ip pref 1 flower chain 0 handle", + "matchCount": "1000000", + "teardown": [ + "$TC qdisc del dev $DEV2 ingress", + "/bin/rm -rf $BATCH_DIR" + ] + }, + { + "id": "14be", + "name": "Concurrently replace same range of 100k flower filters from 10 tc instances", + "category": [ + "filter", + "flower", + "concurrency" + ], + "setup": [ + "/bin/mkdir $BATCH_DIR", + "$TC qdisc add dev $DEV2 ingress", + "./tdc_multibatch.py $DEV2 $BATCH_DIR 100000 1 add", + "$TC -b $BATCH_DIR/add_0", + "./tdc_multibatch.py -d $DEV2 $BATCH_DIR 100000 10 replace" + ], + "cmdUnderTest": "find $BATCH_DIR/replace* -print | xargs -n 1 -P 10 $TC -b", + "expExitCode": "0", + "verifyCmd": "$TC -s filter show dev $DEV2 ingress", + "matchPattern": "filter protocol ip pref 1 flower chain 0 handle", + "matchCount": "100000", + "teardown": [ + "$TC qdisc del dev $DEV2 ingress", + "/bin/rm -rf $BATCH_DIR" + ] + }, + { + "id": "0c44", + "name": "Concurrently delete same range of 100k flower filters from 10 tc instances", + "category": [ + "filter", + "flower", + "concurrency" + ], + "setup": [ + "/bin/mkdir $BATCH_DIR", + "$TC qdisc add dev $DEV2 ingress", + "./tdc_multibatch.py $DEV2 $BATCH_DIR 100000 1 add", + "$TC -b $BATCH_DIR/add_0", + "./tdc_multibatch.py -d $DEV2 $BATCH_DIR 100000 10 del" + ], + "cmdUnderTest": "find $BATCH_DIR/del* -print | xargs -n 1 -P 10 $TC -f -b", + "expExitCode": "123", + "verifyCmd": "$TC -s filter show dev $DEV2 ingress", + "matchPattern": "filter protocol ip pref 1 flower chain 0 handle", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV2 ingress", + "/bin/rm -rf $BATCH_DIR" + ] + }, + { + "id": "ab62", + "name": "Add and delete from same tp with 10 tc instances", + "category": [ + "filter", + "flower", + "concurrency" + ], + "setup": [ + "/bin/mkdir $BATCH_DIR", + "$TC qdisc add dev $DEV2 ingress", + "./tdc_multibatch.py -x init_ $DEV2 $BATCH_DIR 100000 5 add", + "find $BATCH_DIR/init_* -print | xargs -n 1 -P 5 $TC -b", + "./tdc_multibatch.py -x par_ -a 500001 -m 5 $DEV2 $BATCH_DIR 100000 5 add", + "./tdc_multibatch.py -x par_ $DEV2 $BATCH_DIR 100000 5 del" + ], + "cmdUnderTest": "find $BATCH_DIR/par_* -print | xargs -n 1 -P 10 $TC -b", + "expExitCode": "0", + "verifyCmd": "$TC -s filter show dev $DEV2 ingress", + "matchPattern": "filter protocol ip pref 1 flower chain 0 handle", + "matchCount": "500000", + "teardown": [ + "$TC qdisc del dev $DEV2 ingress", + "/bin/rm -rf $BATCH_DIR" + ] + }, + { + "id": "6e8f", + "name": "Replace and delete from same tp with 10 tc instances", + "category": [ + "filter", + "flower", + "concurrency" + ], + "setup": [ + "/bin/mkdir $BATCH_DIR", + "$TC qdisc add dev $DEV2 ingress", + "./tdc_multibatch.py -x init_ $DEV2 $BATCH_DIR 100000 10 add", + "find $BATCH_DIR/init_* -print | xargs -n 1 -P 5 $TC -b", + "./tdc_multibatch.py -x par_ -a 500001 -m 5 $DEV2 $BATCH_DIR 100000 5 replace", + "./tdc_multibatch.py -x par_ $DEV2 $BATCH_DIR 100000 5 del" + ], + "cmdUnderTest": "find $BATCH_DIR/par_* -print | xargs -n 1 -P 10 $TC -b", + "expExitCode": "0", + "verifyCmd": "$TC -s filter show dev $DEV2 ingress", + "matchPattern": "filter protocol ip pref 1 flower chain 0 handle", + "matchCount": "500000", + "teardown": [ + "$TC qdisc del dev $DEV2 ingress", + "/bin/rm -rf $BATCH_DIR" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tdc.py b/tools/testing/selftests/tc-testing/tdc.py index 87a04a8a5945..5cee15659e5f 100755 --- a/tools/testing/selftests/tc-testing/tdc.py +++ b/tools/testing/selftests/tc-testing/tdc.py @@ -23,6 +23,7 @@ from tdc_config import * from tdc_helper import * import TdcPlugin +from TdcResults import * class PluginMgrTestFail(Exception): @@ -60,10 +61,10 @@ class PluginMgr: for pgn_inst in reversed(self.plugin_instances): pgn_inst.post_suite(index) - def call_pre_case(self, test_ordinal, testid): + def call_pre_case(self, testid, test_name, *, test_skip=False): for pgn_inst in self.plugin_instances: try: - pgn_inst.pre_case(test_ordinal, testid) + pgn_inst.pre_case(testid, test_name, test_skip) except Exception as ee: print('exception {} in call to pre_case for {} plugin'. format(ee, pgn_inst.__class__)) @@ -102,7 +103,6 @@ class PluginMgr: self.argparser = argparse.ArgumentParser( description='Linux TC unit tests') - def replace_keywords(cmd): """ For a given executable command, substitute any known @@ -131,12 +131,16 @@ def exec_cmd(args, pm, stage, command): stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=ENVIR) - (rawout, serr) = proc.communicate() - if proc.returncode != 0 and len(serr) > 0: - foutput = serr.decode("utf-8") - else: - foutput = rawout.decode("utf-8") + try: + (rawout, serr) = proc.communicate(timeout=NAMES['TIMEOUT']) + if proc.returncode != 0 and len(serr) > 0: + foutput = serr.decode("utf-8", errors="ignore") + else: + foutput = rawout.decode("utf-8", errors="ignore") + except subprocess.TimeoutExpired: + foutput = "Command \"{}\" timed out\n".format(command) + proc.returncode = 255 proc.stdout.close() proc.stderr.close() @@ -169,6 +173,8 @@ def prepare_env(args, pm, stage, prefix, cmdlist, output = None): file=sys.stderr) print("\n{} *** Error message: \"{}\"".format(prefix, foutput), file=sys.stderr) + print("returncode {}; expected {}".format(proc.returncode, + exit_codes)) print("\n{} *** Aborting test run.".format(prefix), file=sys.stderr) print("\n\n{} *** stdout ***".format(proc.stdout), file=sys.stderr) print("\n\n{} *** stderr ***".format(proc.stderr), file=sys.stderr) @@ -181,26 +187,43 @@ def run_one_test(pm, args, index, tidx): result = True tresult = "" tap = "" + res = TestResult(tidx['id'], tidx['name']) if args.verbose > 0: print("\t====================\n=====> ", end="") print("Test " + tidx["id"] + ": " + tidx["name"]) + if 'skip' in tidx: + if tidx['skip'] == 'yes': + res = TestResult(tidx['id'], tidx['name']) + res.set_result(ResultState.skip) + res.set_errormsg('Test case designated as skipped.') + pm.call_pre_case(tidx['id'], tidx['name'], test_skip=True) + pm.call_post_execute() + return res + # populate NAMES with TESTID for this test NAMES['TESTID'] = tidx['id'] - pm.call_pre_case(index, tidx['id']) + pm.call_pre_case(tidx['id'], tidx['name']) prepare_env(args, pm, 'setup', "-----> prepare stage", tidx["setup"]) if (args.verbose > 0): print('-----> execute stage') pm.call_pre_execute() (p, procout) = exec_cmd(args, pm, 'execute', tidx["cmdUnderTest"]) - exit_code = p.returncode + if p: + exit_code = p.returncode + else: + exit_code = None + pm.call_post_execute() - if (exit_code != int(tidx["expExitCode"])): - result = False - print("exit:", exit_code, int(tidx["expExitCode"])) + if (exit_code is None or exit_code != int(tidx["expExitCode"])): + print("exit: {!r}".format(exit_code)) + print("exit: {}".format(int(tidx["expExitCode"]))) + #print("exit: {!r} {}".format(exit_code, int(tidx["expExitCode"]))) + res.set_result(ResultState.fail) + res.set_failmsg('Command exited with {}, expected {}\n{}'.format(exit_code, tidx["expExitCode"], procout)) print(procout) else: if args.verbose > 0: @@ -211,20 +234,15 @@ def run_one_test(pm, args, index, tidx): if procout: match_index = re.findall(match_pattern, procout) if len(match_index) != int(tidx["matchCount"]): - result = False + res.set_result(ResultState.fail) + res.set_failmsg('Could not match regex pattern. Verify command output:\n{}'.format(procout)) + else: + res.set_result(ResultState.success) elif int(tidx["matchCount"]) != 0: - result = False - - if not result: - tresult += 'not ' - tresult += 'ok {} - {} # {}\n'.format(str(index), tidx['id'], tidx['name']) - tap += tresult - - if result == False: - if procout: - tap += procout + res.set_result(ResultState.fail) + res.set_failmsg('No output generated by verify command.') else: - tap += 'No output!\n' + res.set_result(ResultState.success) prepare_env(args, pm, 'teardown', '-----> teardown stage', tidx['teardown'], procout) pm.call_post_case() @@ -233,7 +251,7 @@ def run_one_test(pm, args, index, tidx): # remove TESTID from NAMES del(NAMES['TESTID']) - return tap + return res def test_runner(pm, args, filtered_tests): """ @@ -253,25 +271,15 @@ def test_runner(pm, args, filtered_tests): emergency_exit = False emergency_exit_message = '' - if args.notap: - if args.verbose: - tap = 'notap requested: omitting test plan\n' - else: - tap = str(index) + ".." + str(tcount) + "\n" + tsr = TestSuiteReport() + try: pm.call_pre_suite(tcount, [tidx['id'] for tidx in testlist]) except Exception as ee: ex_type, ex, ex_tb = sys.exc_info() print('Exception {} {} (caught in pre_suite).'. format(ex_type, ex)) - # when the extra print statements are uncommented, - # the traceback does not appear between them - # (it appears way earlier in the tdc.py output) - # so don't bother ... - # print('--------------------(') - # print('traceback') traceback.print_tb(ex_tb) - # print('--------------------)') emergency_exit_message = 'EMERGENCY EXIT, call_pre_suite failed with exception {} {}\n'.format(ex_type, ex) emergency_exit = True stage = 'pre-SUITE' @@ -287,15 +295,26 @@ def test_runner(pm, args, filtered_tests): if args.verbose > 1: print('Not executing test {} {} because DEV2 not defined'. format(tidx['id'], tidx['name'])) + res = TestResult(tidx['id'], tidx['name']) + res.set_result(ResultState.skip) + res.set_errormsg('Not executed because DEV2 is not defined') + tsr.add_resultdata(res) continue try: badtest = tidx # in case it goes bad - tap += run_one_test(pm, args, index, tidx) + res = run_one_test(pm, args, index, tidx) + tsr.add_resultdata(res) except PluginMgrTestFail as pmtf: ex_type, ex, ex_tb = sys.exc_info() stage = pmtf.stage message = pmtf.message output = pmtf.output + res = TestResult(tidx['id'], tidx['name']) + res.set_result(ResultState.skip) + res.set_errormsg(pmtf.message) + res.set_failmsg(pmtf.output) + tsr.add_resultdata(res) + index += 1 print(message) print('Exception {} {} (caught in test_runner, running test {} {} {} stage {})'. format(ex_type, ex, index, tidx['id'], tidx['name'], stage)) @@ -314,16 +333,16 @@ def test_runner(pm, args, filtered_tests): # if we failed in setup or teardown, # fill in the remaining tests with ok-skipped count = index - if not args.notap: - tap += 'about to flush the tap output if tests need to be skipped\n' - if tcount + 1 != index: - for tidx in testlist[index - 1:]: - msg = 'skipped - previous {} failed'.format(stage) - tap += 'ok {} - {} # {} {} {}\n'.format( - count, tidx['id'], msg, index, badtest.get('id', '--Unknown--')) - count += 1 - tap += 'done flushing skipped test tap output\n' + if tcount + 1 != count: + for tidx in testlist[count - 1:]: + res = TestResult(tidx['id'], tidx['name']) + res.set_result(ResultState.skip) + msg = 'skipped - previous {} failed {} {}'.format(stage, + index, badtest.get('id', '--Unknown--')) + res.set_errormsg(msg) + tsr.add_resultdata(res) + count += 1 if args.pause: print('Want to pause\nPress enter to continue ...') @@ -332,7 +351,7 @@ def test_runner(pm, args, filtered_tests): pm.call_post_suite(index) - return tap + return tsr def has_blank_ids(idlist): """ @@ -373,6 +392,10 @@ def set_args(parser): Set the command line arguments for tdc. """ parser.add_argument( + '--outfile', type=str, + help='Path to the file in which results should be saved. ' + + 'Default target is the current directory.') + parser.add_argument( '-p', '--path', type=str, help='The full path to the tc executable to use') sg = parser.add_argument_group( @@ -408,8 +431,9 @@ def set_args(parser): '-v', '--verbose', action='count', default=0, help='Show the commands that are being run') parser.add_argument( - '-N', '--notap', action='store_true', - help='Suppress tap results for command under test') + '--format', default='tap', const='tap', nargs='?', + choices=['none', 'xunit', 'tap'], + help='Specify the format for test results. (Default: TAP)') parser.add_argument('-d', '--device', help='Execute the test case in flower category') parser.add_argument( @@ -430,6 +454,8 @@ def check_default_settings(args, remaining, pm): NAMES['TC'] = args.path if args.device != None: NAMES['DEV2'] = args.device + if 'TIMEOUT' not in NAMES: + NAMES['TIMEOUT'] = None if not os.path.isfile(NAMES['TC']): print("The specified tc path " + NAMES['TC'] + " does not exist.") exit(1) @@ -624,12 +650,30 @@ def set_operation_mode(pm, args): if len(alltests): catresults = test_runner(pm, args, alltests) + if args.format == 'none': + print('Test results output suppression requested\n') + else: + print('\nAll test results: \n') + if args.format == 'xunit': + suffix = 'xml' + res = catresults.format_xunit() + elif args.format == 'tap': + suffix = 'tap' + res = catresults.format_tap() + print(res) + print('\n\n') + if not args.outfile: + fname = 'test-results.{}'.format(suffix) + else: + fname = args.outfile + with open(fname, 'w') as fh: + fh.write(res) + fh.close() + if os.getenv('SUDO_UID') is not None: + os.chown(fname, uid=int(os.getenv('SUDO_UID')), + gid=int(os.getenv('SUDO_GID'))) else: - catresults = 'No tests found\n' - if args.notap: - print('Tap output suppression requested\n') - else: - print('All test results: \n\n{}'.format(catresults)) + print('No tests found\n') def main(): """ diff --git a/tools/testing/selftests/tc-testing/tdc_batch.py b/tools/testing/selftests/tc-testing/tdc_batch.py index 52fa539dc662..6a2bd2cf528e 100755 --- a/tools/testing/selftests/tc-testing/tdc_batch.py +++ b/tools/testing/selftests/tc-testing/tdc_batch.py @@ -13,6 +13,12 @@ parser.add_argument("device", help="device name") parser.add_argument("file", help="batch file name") parser.add_argument("-n", "--number", type=int, help="how many lines in batch file") +parser.add_argument( + "-a", + "--handle_start", + type=int, + default=1, + help="start handle range from (default: 1)") parser.add_argument("-o", "--skip_sw", help="skip_sw (offload), by default skip_hw", action="store_true") @@ -22,6 +28,21 @@ parser.add_argument("-s", "--share_action", parser.add_argument("-p", "--prio", help="all filters have different prio", action="store_true") +parser.add_argument( + "-e", + "--operation", + choices=['add', 'del', 'replace'], + default='add', + help="operation to perform on filters" + "(default: add filter)") +parser.add_argument( + "-m", + "--mac_prefix", + type=int, + default=0, + choices=range(0, 256), + help="third byte of source MAC address of flower filter" + "(default: 0)") args = parser.parse_args() device = args.device @@ -31,6 +52,8 @@ number = 1 if args.number: number = args.number +handle_start = args.handle_start + skip = "skip_hw" if args.skip_sw: skip = "skip_sw" @@ -45,16 +68,43 @@ if args.prio: if number > 0x4000: number = 0x4000 +mac_prefix = args.mac_prefix + +def format_add_filter(device, prio, handle, skip, src_mac, dst_mac, + share_action): + return ("filter add dev {} {} protocol ip parent ffff: handle {} " + " flower {} src_mac {} dst_mac {} action drop {}".format( + device, prio, handle, skip, src_mac, dst_mac, share_action)) + + +def format_rep_filter(device, prio, handle, skip, src_mac, dst_mac, + share_action): + return ("filter replace dev {} {} protocol ip parent ffff: handle {} " + " flower {} src_mac {} dst_mac {} action drop {}".format( + device, prio, handle, skip, src_mac, dst_mac, share_action)) + + +def format_del_filter(device, prio, handle, skip, src_mac, dst_mac, + share_action): + return ("filter del dev {} {} protocol ip parent ffff: handle {} " + "flower".format(device, prio, handle)) + + +formatter = format_add_filter +if args.operation == "del": + formatter = format_del_filter +elif args.operation == "replace": + formatter = format_rep_filter + index = 0 for i in range(0x100): for j in range(0x100): for k in range(0x100): mac = ("{:02x}:{:02x}:{:02x}".format(i, j, k)) - src_mac = "e4:11:00:" + mac + src_mac = "e4:11:{:02x}:{}".format(mac_prefix, mac) dst_mac = "e4:12:00:" + mac - cmd = ("filter add dev {} {} protocol ip parent ffff: flower {} " - "src_mac {} dst_mac {} action drop {}".format - (device, prio, skip, src_mac, dst_mac, share_action)) + cmd = formatter(device, prio, handle_start + index, skip, src_mac, + dst_mac, share_action) file.write("{}\n".format(cmd)) index += 1 if index >= number: diff --git a/tools/testing/selftests/tc-testing/tdc_config.py b/tools/testing/selftests/tc-testing/tdc_config.py index d651bc1501bd..942c70c041be 100644 --- a/tools/testing/selftests/tc-testing/tdc_config.py +++ b/tools/testing/selftests/tc-testing/tdc_config.py @@ -15,6 +15,9 @@ NAMES = { 'DEV1': 'v0p1', 'DEV2': '', 'BATCH_FILE': './batch.txt', + 'BATCH_DIR': 'tmp', + # Length of time in seconds to wait before terminating a command + 'TIMEOUT': 12, # Name of the namespace to use 'NS': 'tcut', # Directory containing eBPF test programs diff --git a/tools/testing/selftests/tc-testing/tdc_multibatch.py b/tools/testing/selftests/tc-testing/tdc_multibatch.py new file mode 100755 index 000000000000..5e7237952e49 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tdc_multibatch.py @@ -0,0 +1,65 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: GPL-2.0 +""" +tdc_multibatch.py - a thin wrapper over tdc_batch.py to generate multiple batch +files + +Copyright (C) 2019 Vlad Buslov <vladbu@mellanox.com> +""" + +import argparse +import os + +parser = argparse.ArgumentParser( + description='TC multiple batch file generator') +parser.add_argument("device", help="device name") +parser.add_argument("dir", help="where to put batch files") +parser.add_argument( + "num_filters", type=int, help="how many lines per batch file") +parser.add_argument("num_files", type=int, help="how many batch files") +parser.add_argument( + "operation", + choices=['add', 'del', 'replace'], + help="operation to perform on filters") +parser.add_argument( + "-x", + "--file_prefix", + default="", + help="prefix for generated batch file names") +parser.add_argument( + "-d", + "--duplicate_handles", + action="store_true", + help="duplicate filter handle range in all files") +parser.add_argument( + "-a", + "--handle_start", + type=int, + default=1, + help="start handle range from (default: 1)") +parser.add_argument( + "-m", + "--mac_prefix", + type=int, + default=0, + choices=range(0, 256), + help="add this value to third byte of source MAC address of flower filter" + "(default: 0)") +args = parser.parse_args() + +device = args.device +dir = args.dir +file_prefix = args.file_prefix + args.operation + "_" +num_filters = args.num_filters +num_files = args.num_files +operation = args.operation +duplicate_handles = args.duplicate_handles +handle = args.handle_start +mac_prefix = args.mac_prefix + +for i in range(num_files): + file = dir + '/' + file_prefix + str(i) + os.system("./tdc_batch.py -n {} -a {} -e {} -m {} {} {}".format( + num_filters, handle, operation, i + mac_prefix, device, file)) + if not duplicate_handles: + handle += num_filters |
