ftracetest: Add POSIX.3 standard and XFAIL result codes

Add XFAIL and POSIX 1003.3 standard codes (UNRESOLVED/
UNTESTED/UNSUPPORTED) as result codes. These are used for the
results that test case is expected to fail or unsupported
feature (by config).

To return these result code, this introduces exit_unresolved,
exit_untested, exit_unsupported and exit_xfail functions,
which use real-time signals to notify the result code to
ftracetest.

This also set "errexit" option for the testcases, so that
the tests don't need to exit explicitly.

Note that if the test returns UNRESOLVED/UNSUPPORTED/FAIL,
its test log including executed commands is shown on console
and main logfile as below.

  ------
  # ./ftracetest samples/
  === Ftrace unit tests ===
  [1] failure-case example        [FAIL]
  execute: /home/fedora/ksrc/linux-3/tools/testing/selftests/ftrace/samples/fail.tc
  + . /home/fedora/ksrc/linux-3/tools/testing/selftests/ftrace/samples/fail.tc
  ++ cat non-exist-file
  cat: non-exist-file: No such file or directory
  [2] pass-case example   [PASS]
  [3] unresolved-case example     [UNRESOLVED]
  execute: /home/fedora/ksrc/linux-3/tools/testing/selftests/ftrace/samples/unresolved.tc
  + . /home/fedora/ksrc/linux-3/tools/testing/selftests/ftrace/samples/unresolved.tc
  ++ trap exit_unresolved INT
  ++ kill -INT 29324
  +++ exit_unresolved
  +++ kill -s 38 29265
  +++ exit 0
  [4] unsupported-case example    [UNSUPPORTED]
  execute: /home/fedora/ksrc/linux-3/tools/testing/selftests/ftrace/samples/unsupported.tc
  + . /home/fedora/ksrc/linux-3/tools/testing/selftests/ftrace/samples/unsupported.tc
  ++ exit_unsupported
  ++ kill -s 40 29265
  ++ exit 0
  [5] untested-case example       [UNTESTED]
  [6] xfail-case example  [XFAIL]

  # of passed:  1
  # of failed:  1
  # of unresolved:  1
  # of untested:  1
  # of unsupported:  1
  # of xfailed:  1
  # of undefined(test bug):  0
  ------

Link: http://lkml.kernel.org/p/20140929120211.30203.99510.stgit@kbuild-f20.novalocal

Acked-by: Namhyung Kim <namhyung@kernel.org>
Signed-off-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>

5 files changed, 23 insertions, 18 deletions

diff --git a/tools/testing/selftests/ftrace/test.d/00basic/basic2.tc b/tools/testing/selftests/ftrace/test.d/00basic/basic2.tc
index b04f30df0db3..bf9a7b037924 100644
--- a/tools/testing/selftests/ftrace/test.d/00basic/basic2.tc
+++ b/tools/testing/selftests/ftrace/test.d/00basic/basic2.tc
@@ -1,6 +1,7 @@
 #!/bin/sh
 # description: Basic test for tracers
+test -f available_tracers
 for t in `cat available_tracers`; do
-  echo $t > current_tracer || exit 1
+  echo $t > current_tracer
 done
 echo nop > current_tracer
diff --git a/tools/testing/selftests/ftrace/test.d/00basic/basic3.tc b/tools/testing/selftests/ftrace/test.d/00basic/basic3.tc
index 0c1a3a207636..bde6625d9785 100644
--- a/tools/testing/selftests/ftrace/test.d/00basic/basic3.tc
+++ b/tools/testing/selftests/ftrace/test.d/00basic/basic3.tc
@@ -1,8 +1,8 @@
 #!/bin/sh
 # description: Basic trace clock test
-[ -f trace_clock ] || exit 1
+test -f trace_clock
 for c in `cat trace_clock | tr  -d \[\]`; do
-  echo $c > trace_clock || exit 1
-  grep '\['$c'\]' trace_clock || exit 1
+  echo $c > trace_clock
+  grep '\['$c'\]' trace_clock
 done
 echo local > trace_clock
diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/add_and_remove.tc b/tools/testing/selftests/ftrace/test.d/kprobe/add_and_remove.tc
index 5ddfb476eceb..1b8b665ab2b3 100644
--- a/tools/testing/selftests/ftrace/test.d/kprobe/add_and_remove.tc
+++ b/tools/testing/selftests/ftrace/test.d/kprobe/add_and_remove.tc
@@ -1,11 +1,11 @@
 #!/bin/sh
 # description: Kprobe dynamic event - adding and removing
 
-[ -f kprobe_events ] || exit 1
+[ -f kprobe_events ] || exit_unsupported # this is configurable
 
-echo 0 > events/enable || exit 1
-echo > kprobe_events || exit 1
-echo p:myevent do_fork > kprobe_events || exit 1
-grep myevent kprobe_events || exit 1
-[ -d events/kprobes/myevent ] || exit 1
+echo 0 > events/enable
+echo > kprobe_events
+echo p:myevent do_fork > kprobe_events
+grep myevent kprobe_events
+test -d events/kprobes/myevent
 echo > kprobe_events
diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/busy_check.tc b/tools/testing/selftests/ftrace/test.d/kprobe/busy_check.tc
index 588fde97e93f..b55c84003587 100644
--- a/tools/testing/selftests/ftrace/test.d/kprobe/busy_check.tc
+++ b/tools/testing/selftests/ftrace/test.d/kprobe/busy_check.tc
@@ -1,14 +1,13 @@
 #!/bin/sh
 # description: Kprobe dynamic event - busy event check
 
-[ -f kprobe_events ] || exit 1
+[ -f kprobe_events ] || exit_unsupported
 
-echo 0 > events/enable || exit 1
-echo > kprobe_events || exit 1
-echo p:myevent do_fork > kprobe_events || exit 1
-[ -d events/kprobes/myevent ] || exit 1
-echo 1 > events/kprobes/myevent/enable || exit 1
+echo 0 > events/enable
+echo > kprobe_events
+echo p:myevent do_fork > kprobe_events
+test -d events/kprobes/myevent
+echo 1 > events/kprobes/myevent/enable
 echo > kprobe_events && exit 1 # this must fail
-echo 0 > events/kprobes/myevent/enable || exit 1
+echo 0 > events/kprobes/myevent/enable
 echo > kprobe_events # this must succeed
-
diff --git a/tools/testing/selftests/ftrace/test.d/template b/tools/testing/selftests/ftrace/test.d/template
index ce5f735b2e65..5448f7abad5f 100644
--- a/tools/testing/selftests/ftrace/test.d/template
+++ b/tools/testing/selftests/ftrace/test.d/template
@@ -1,4 +1,9 @@
 #!/bin/sh
 # description: %HERE DESCRIBE WHAT THIS DOES%
 # you have to add ".tc" extention for your testcase file
+# Note that all tests are run with "errexit" option.
+
 exit 0 # Return 0 if the test is passed, otherwise return !0
+# If the test could not run because of lack of feature, call exit_unsupported
+# If the test returned unclear results, call exit_unresolved
+# If the test is a dummy, or a placeholder, call exit_untested