From 7cc5565e7c30bbfaa2ebc77a18ab9355137dfa5e Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 2 Aug 2018 18:16:33 +0100 Subject: [PATCH 1/8] testutils: Correctly print incomplete tests as "not ok # TODO" The TAP specification says that failing tests that are currently expected to fail (like Automake's XFAIL) are to be reported as "not ok", with that failure ignored as a result of the TODO directive, with this example: not ok 3 - infinite loop # TODO halting problem unsolved A test reported as "ok # TODO" indicates that something that is expected to fail has unexpectedly succeeded, similar to Automake's XPASS. Signed-off-by: Simon McVittie --- glib/gtestutils.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/glib/gtestutils.c b/glib/gtestutils.c index da6c7338c..fde5281e4 100644 --- a/glib/gtestutils.c +++ b/glib/gtestutils.c @@ -959,7 +959,20 @@ g_test_log (GTestLogType lbit, fail = result == G_TEST_RUN_FAILURE; if (test_tap_log) { - g_print ("%s %d %s", fail ? "not ok" : "ok", test_run_count, string1); + const gchar *ok; + + /* The TAP representation for an expected failure starts with + * "not ok", even though it does not actually count as failing + * due to the use of the TODO directive. "ok # TODO" would mean + * a test that was expected to fail unexpectedly succeeded, + * for which GTestResult does not currently have a + * representation. */ + if (fail || result == G_TEST_RUN_INCOMPLETE) + ok = "not ok"; + else + ok = "ok"; + + g_print ("%s %d %s", ok, test_run_count, string1); if (result == G_TEST_RUN_INCOMPLETE) g_print (" # TODO %s\n", string2 ? string2 : ""); else if (result == G_TEST_RUN_SKIPPED) From 49a877764d818ffeabe065bcc37ed22fbe6b6b51 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 3 Aug 2018 13:42:38 +0100 Subject: [PATCH 2/8] testing: Clarify why incomplete tests cause g_test_failed() Signed-off-by: Simon McVittie --- glib/tests/testing.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/glib/tests/testing.c b/glib/tests/testing.c index de95f2635..0c2cf5d2a 100644 --- a/glib/tests/testing.c +++ b/glib/tests/testing.c @@ -648,7 +648,10 @@ test_incomplete (void) if (g_test_subprocess ()) { g_test_incomplete ("not done"); - g_assert (g_test_failed ()); + /* This function really means "the test concluded with a non-successful + * status" rather than "the test failed": it is documented to return + * true for skipped and incomplete tests, not just for failures. */ + g_assert_true (g_test_failed ()); return; } g_test_trap_subprocess (NULL, 0, 0); From ffe0402e0227808813ba0d8d279d18d3da88c704 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 3 Aug 2018 13:42:57 +0100 Subject: [PATCH 3/8] testing: Assert that skipped tests cause g_test_failed() Signed-off-by: Simon McVittie --- glib/tests/testing.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/glib/tests/testing.c b/glib/tests/testing.c index 0c2cf5d2a..f2870da0c 100644 --- a/glib/tests/testing.c +++ b/glib/tests/testing.c @@ -622,6 +622,10 @@ static void test_skip (void) { g_test_skip ("Skipped should count as passed, not failed"); + /* This function really means "the test concluded with a non-successful + * status" rather than "the test failed": it is documented to return + * true for skipped and incomplete tests, not just for failures. */ + g_assert_true (g_test_failed ()); } static void From ba3b4421521128d43a484dc58dc9415306684e1c Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 3 Aug 2018 13:44:51 +0100 Subject: [PATCH 4/8] testing: Use g_assert_true(), which is preferred for unit tests In the test for the unit testing framework, we might as well have exemplary behaviour. Leave behind a single call to g_assert (TRUE) just to prove that it still works. Signed-off-by: Simon McVittie --- glib/tests/testing.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/glib/tests/testing.c b/glib/tests/testing.c index f2870da0c..859105f48 100644 --- a/glib/tests/testing.c +++ b/glib/tests/testing.c @@ -78,6 +78,7 @@ test_assertions (void) g_assert_cmpfloat_with_epsilon (3.14, 3.15, 0.01); g_assert_cmpfloat_with_epsilon (3.14159, 3.1416, 0.0001); g_assert (TRUE); + g_assert_true (TRUE); g_assert_cmpstr ("foo", !=, "faa"); fuu = g_strdup_printf ("f%s", "uu"); g_test_queue_free (fuu); @@ -170,7 +171,7 @@ test_fork_timeout (void) g_usleep (1000 * 1000); } g_test_trap_assert_failed(); - g_assert (g_test_trap_reached_timeout()); + g_assert_true (g_test_trap_reached_timeout()); } G_GNUC_END_IGNORE_DEPRECATIONS @@ -233,7 +234,7 @@ test_subprocess_timeout (void) /* allow child to run for only a fraction of a second */ g_test_trap_subprocess (NULL, 0.11 * 1000000, 0); g_test_trap_assert_failed (); - g_assert (g_test_trap_reached_timeout ()); + g_assert_true (g_test_trap_reached_timeout ()); } /* run a test with fixture setup and teardown */ @@ -246,7 +247,7 @@ static void fixturetest_setup (Fixturetest *fix, gconstpointer test_data) { - g_assert (test_data == (void*) 0xc0cac01a); + g_assert_true (test_data == (void*) 0xc0cac01a); fix->seed = 18; fix->prime = 19; fix->msg = g_strdup_printf ("%d", fix->prime); @@ -259,13 +260,13 @@ fixturetest_test (Fixturetest *fix, g_assert_cmpint (prime, ==, fix->prime); prime = g_ascii_strtoull (fix->msg, NULL, 0); g_assert_cmpint (prime, ==, fix->prime); - g_assert (test_data == (void*) 0xc0cac01a); + g_assert_true (test_data == (void*) 0xc0cac01a); } static void fixturetest_teardown (Fixturetest *fix, gconstpointer test_data) { - g_assert (test_data == (void*) 0xc0cac01a); + g_assert_true (test_data == (void*) 0xc0cac01a); g_free (fix->msg); } @@ -308,7 +309,7 @@ test_rand2 (void) static void test_data_test (gconstpointer test_data) { - g_assert (test_data == (void*) 0xc0c0baba); + g_assert_true (test_data == (void*) 0xc0c0baba); } static void @@ -319,7 +320,7 @@ test_random_conversions (void) char *err, *str = g_strdup_printf ("%d", vint); gint64 vint64 = g_ascii_strtoll (str, &err, 10); g_assert_cmphex (vint, ==, vint64); - g_assert (!err || *err == 0); + g_assert_true (!err || *err == 0); g_free (str); } @@ -639,7 +640,7 @@ test_fail (void) if (g_test_subprocess ()) { g_test_fail (); - g_assert (g_test_failed ()); + g_assert_true (g_test_failed ()); return; } g_test_trap_subprocess (NULL, 0, 0); @@ -671,7 +672,7 @@ test_subprocess_timed_out (void) return; } g_test_trap_subprocess (NULL, 50000, 0); - g_assert (g_test_trap_reached_timeout ()); + g_assert_true (g_test_trap_reached_timeout ()); } static const char *argv0; From bbefa739976f3c3d9a43e661571e698526cf6b43 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 3 Aug 2018 13:51:53 +0100 Subject: [PATCH 5/8] testutils: Treat incomplete tests more like skipped tests If a test is marked with g_test_incomplete(), then it is expected to fail, so when it fails the test executable should still exit 0 (or possibly 77, if all tests are either skipped or incomplete). Signed-off-by: Simon McVittie --- glib/gtestutils.c | 5 +++-- glib/tests/testing.c | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/glib/gtestutils.c b/glib/gtestutils.c index fde5281e4..686a4cad3 100644 --- a/glib/gtestutils.c +++ b/glib/gtestutils.c @@ -990,7 +990,7 @@ g_test_log (GTestLogType lbit, g_print ("Bail out!\n"); g_abort (); } - if (result == G_TEST_RUN_SKIPPED) + if (result == G_TEST_RUN_SKIPPED || result == G_TEST_RUN_INCOMPLETE) test_skipped_count++; break; case G_TEST_LOG_MIN_RESULT: @@ -2338,7 +2338,8 @@ test_case_run (GTestCase *tc) test_uri_base = old_base; return (success == G_TEST_RUN_SUCCESS || - success == G_TEST_RUN_SKIPPED); + success == G_TEST_RUN_SKIPPED || + success == G_TEST_RUN_INCOMPLETE); } static gboolean diff --git a/glib/tests/testing.c b/glib/tests/testing.c index 859105f48..aad8b6317 100644 --- a/glib/tests/testing.c +++ b/glib/tests/testing.c @@ -660,6 +660,11 @@ test_incomplete (void) return; } g_test_trap_subprocess (NULL, 0, 0); + /* An incomplete test represents functionality that is known not to be + * implemented yet (an expected failure), so it does not cause test + * failure; but it does count as the test having been skipped, which + * causes nonzero exit status 77, which is treated as failure by + * g_test_trap_subprocess(). */ g_test_trap_assert_failed (); } From 0a180bb51b200f0df2084509b0f9d23d73a29522 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 3 Aug 2018 14:30:01 +0100 Subject: [PATCH 6/8] testing: Assert that test-case results are combined correctly This repurposes the existing skip-all test as "combining", since it is no longer entirely about runs where every test-case was skipped. Signed-off-by: Simon McVittie --- glib/tests/testing.c | 174 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 158 insertions(+), 16 deletions(-) diff --git a/glib/tests/testing.c b/glib/tests/testing.c index aad8b6317..64c5244f9 100644 --- a/glib/tests/testing.c +++ b/glib/tests/testing.c @@ -634,29 +634,44 @@ test_pass (void) { } +static void +subprocess_fail (void) +{ + /* Exit 1 instead of raising SIGABRT so that we can make assertions about + * how this combines with skipped/incomplete tests */ + g_test_set_nonfatal_assertions (); + g_test_fail (); + g_assert_true (g_test_failed ()); +} + static void test_fail (void) { if (g_test_subprocess ()) { - g_test_fail (); - g_assert_true (g_test_failed ()); + subprocess_fail (); return; } g_test_trap_subprocess (NULL, 0, 0); g_test_trap_assert_failed (); } +static void +subprocess_incomplete (void) +{ + g_test_incomplete ("not done"); + /* This function really means "the test concluded with a non-successful + * status" rather than "the test failed": it is documented to return + * true for skipped and incomplete tests, not just for failures. */ + g_assert_true (g_test_failed ()); +} + static void test_incomplete (void) { if (g_test_subprocess ()) { - g_test_incomplete ("not done"); - /* This function really means "the test concluded with a non-successful - * status" rather than "the test failed": it is documented to return - * true for skipped and incomplete tests, not just for failures. */ - g_assert_true (g_test_failed ()); + subprocess_incomplete (); return; } g_test_trap_subprocess (NULL, 0, 0); @@ -683,12 +698,13 @@ test_subprocess_timed_out (void) static const char *argv0; static void -test_skip_all (void) +test_combining (void) { GPtrArray *argv; GError *error = NULL; int status; + g_test_message ("single test case skipped -> overall status 77"); argv = g_ptr_array_new (); g_ptr_array_add (argv, (char *) argv0); g_ptr_array_add (argv, "--GTestSubprocess"); @@ -706,15 +722,16 @@ test_skip_all (void) g_assert_error (error, G_SPAWN_EXIT_ERROR, 77); g_clear_error (&error); + g_test_message ("each test case skipped -> overall status 77"); g_ptr_array_set_size (argv, 0); g_ptr_array_add (argv, (char *) argv0); g_ptr_array_add (argv, "--GTestSubprocess"); g_ptr_array_add (argv, "-p"); g_ptr_array_add (argv, "/misc/skip"); g_ptr_array_add (argv, "-p"); - g_ptr_array_add (argv, "/misc/skip-all/subprocess/skip1"); + g_ptr_array_add (argv, "/misc/combining/subprocess/skip1"); g_ptr_array_add (argv, "-p"); - g_ptr_array_add (argv, "/misc/skip-all/subprocess/skip2"); + g_ptr_array_add (argv, "/misc/combining/subprocess/skip2"); g_ptr_array_add (argv, NULL); g_spawn_sync (NULL, (char **) argv->pdata, NULL, @@ -727,15 +744,34 @@ test_skip_all (void) g_assert_error (error, G_SPAWN_EXIT_ERROR, 77); g_clear_error (&error); + g_test_message ("single test case incomplete -> overall status 77"); + g_ptr_array_set_size (argv, 0); + g_ptr_array_add (argv, (char *) argv0); + g_ptr_array_add (argv, "--GTestSubprocess"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/misc/combining/subprocess/incomplete"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, NULL, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_exit_status (status, &error); + g_assert_error (error, G_SPAWN_EXIT_ERROR, 77); + g_clear_error (&error); + + g_test_message ("one pass and some skipped -> overall status 0"); g_ptr_array_set_size (argv, 0); g_ptr_array_add (argv, (char *) argv0); g_ptr_array_add (argv, "--GTestSubprocess"); g_ptr_array_add (argv, "-p"); g_ptr_array_add (argv, "/misc/skip"); g_ptr_array_add (argv, "-p"); - g_ptr_array_add (argv, "/misc/skip-all/subprocess/pass"); + g_ptr_array_add (argv, "/misc/combining/subprocess/pass"); g_ptr_array_add (argv, "-p"); - g_ptr_array_add (argv, "/misc/skip-all/subprocess/skip1"); + g_ptr_array_add (argv, "/misc/combining/subprocess/skip1"); g_ptr_array_add (argv, NULL); g_spawn_sync (NULL, (char **) argv->pdata, NULL, @@ -747,6 +783,110 @@ test_skip_all (void) g_spawn_check_exit_status (status, &error); g_assert_no_error (error); + g_test_message ("one pass and some incomplete -> overall status 0"); + g_ptr_array_set_size (argv, 0); + g_ptr_array_add (argv, (char *) argv0); + g_ptr_array_add (argv, "--GTestSubprocess"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/misc/combining/subprocess/pass"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/misc/combining/subprocess/incomplete"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, NULL, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_exit_status (status, &error); + g_assert_no_error (error); + + g_test_message ("one pass and mix of skipped and incomplete -> overall status 0"); + g_ptr_array_set_size (argv, 0); + g_ptr_array_add (argv, (char *) argv0); + g_ptr_array_add (argv, "--GTestSubprocess"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/misc/combining/subprocess/pass"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/misc/combining/subprocess/skip1"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/misc/combining/subprocess/incomplete"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, NULL, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_exit_status (status, &error); + g_assert_no_error (error); + + g_test_message ("one fail and some skipped -> overall status fail"); + g_ptr_array_set_size (argv, 0); + g_ptr_array_add (argv, (char *) argv0); + g_ptr_array_add (argv, "--GTestSubprocess"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/misc/skip"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/misc/combining/subprocess/fail"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/misc/combining/subprocess/skip1"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, NULL, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_exit_status (status, &error); + g_assert_error (error, G_SPAWN_EXIT_ERROR, 1); + g_clear_error (&error); + + g_test_message ("one fail and some incomplete -> overall status fail"); + g_ptr_array_set_size (argv, 0); + g_ptr_array_add (argv, (char *) argv0); + g_ptr_array_add (argv, "--GTestSubprocess"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/misc/combining/subprocess/fail"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/misc/combining/subprocess/incomplete"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, NULL, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_exit_status (status, &error); + g_assert_error (error, G_SPAWN_EXIT_ERROR, 1); + g_clear_error (&error); + + g_test_message ("one fail and mix of skipped and incomplete -> overall status fail"); + g_ptr_array_set_size (argv, 0); + g_ptr_array_add (argv, (char *) argv0); + g_ptr_array_add (argv, "--GTestSubprocess"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/misc/combining/subprocess/fail"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/misc/combining/subprocess/skip1"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/misc/combining/subprocess/incomplete"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, NULL, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_exit_status (status, &error); + g_assert_error (error, G_SPAWN_EXIT_ERROR, 1); + g_clear_error (&error); + g_ptr_array_unref (argv); } @@ -814,10 +954,12 @@ main (int argc, g_test_add_func ("/misc/nonfatal", test_nonfatal); g_test_add_func ("/misc/skip", test_skip); - g_test_add_func ("/misc/skip-all", test_skip_all); - g_test_add_func ("/misc/skip-all/subprocess/skip1", test_skip); - g_test_add_func ("/misc/skip-all/subprocess/skip2", test_skip); - g_test_add_func ("/misc/skip-all/subprocess/pass", test_pass); + g_test_add_func ("/misc/combining", test_combining); + g_test_add_func ("/misc/combining/subprocess/fail", subprocess_fail); + g_test_add_func ("/misc/combining/subprocess/skip1", test_skip); + g_test_add_func ("/misc/combining/subprocess/skip2", test_skip); + g_test_add_func ("/misc/combining/subprocess/incomplete", subprocess_incomplete); + g_test_add_func ("/misc/combining/subprocess/pass", test_pass); g_test_add_func ("/misc/fail", test_fail); g_test_add_func ("/misc/incomplete", test_incomplete); g_test_add_func ("/misc/timeout", test_subprocess_timed_out); From 54a5f37f120be3577eab4a2b806c4ff14ea33ea3 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 3 Aug 2018 14:33:52 +0100 Subject: [PATCH 7/8] g_test_run: Document g_test_incomplete's similarity to g_test_skip Signed-off-by: Simon McVittie --- glib/gtestutils.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/glib/gtestutils.c b/glib/gtestutils.c index 686a4cad3..5290572be 100644 --- a/glib/gtestutils.c +++ b/glib/gtestutils.c @@ -1733,11 +1733,13 @@ g_test_get_root (void) * particular code runs before or after a given test case, use * g_test_add(), which lets you specify setup and teardown functions. * - * If all tests are skipped, this function will return 0 if - * producing TAP output, or 77 (treated as "skip test" by Automake) otherwise. + * If all tests are skipped or marked as incomplete (expected failures), + * this function will return 0 if producing TAP output, or 77 (treated + * as "skip test" by Automake) otherwise. * * Returns: 0 on success, 1 on failure (assuming it returns at all), - * 0 or 77 if all tests were skipped with g_test_skip() + * 0 or 77 if all tests were skipped with g_test_skip() and/or + * g_test_incomplete() * * Since: 2.16 */ From b8d751af3e611d4c1ee9e2ba511f1725a7ddcd83 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 3 Aug 2018 15:09:28 +0100 Subject: [PATCH 8/8] testing: Exercise TAP output (Tweaked by Philip Withnall to fix some minor leaks, code formatting, and add a test comment.) Signed-off-by: Simon McVittie --- glib/tests/Makefile.am | 1 + glib/tests/meson.build | 7 ++ glib/tests/testing-helper.c | 97 +++++++++++++++++++++++++++ glib/tests/testing.c | 126 ++++++++++++++++++++++++++++++++++++ 4 files changed, 231 insertions(+) create mode 100644 glib/tests/testing-helper.c diff --git a/glib/tests/Makefile.am b/glib/tests/Makefile.am index 172d16607..3f1050dc9 100644 --- a/glib/tests/Makefile.am +++ b/glib/tests/Makefile.am @@ -38,6 +38,7 @@ dist_test_extra_scripts = \ test_extra_programs = \ test-spawn-echo \ + testing-helper \ $(NULL) test_programs = \ diff --git a/glib/tests/meson.build b/glib/tests/meson.build index f5c0fbecb..0af71b072 100644 --- a/glib/tests/meson.build +++ b/glib/tests/meson.build @@ -183,6 +183,13 @@ executable('test-spawn-echo', 'test-spawn-echo.c', install: installed_tests_enabled, ) +executable('testing-helper', 'testing-helper.c', + c_args : test_cargs, + dependencies : test_deps, + install_dir: installed_tests_execdir, + install: installed_tests_enabled, +) + # some testing of gtester functionality if not meson.is_cross_build() and host_system != 'windows' xmllint = find_program('xmllint', required: false) diff --git a/glib/tests/testing-helper.c b/glib/tests/testing-helper.c new file mode 100644 index 000000000..43127e8dd --- /dev/null +++ b/glib/tests/testing-helper.c @@ -0,0 +1,97 @@ +/* + * Copyright 2018 Collabora Ltd. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General + * Public License along with this library; if not, see + * . + */ + +#include + +static void +test_pass (void) +{ +} + +static void +test_skip (void) +{ + g_test_skip ("not enough tea"); +} + +static void +test_fail (void) +{ + g_test_fail (); +} + +static void +test_incomplete (void) +{ + g_test_incomplete ("mind reading not implemented yet"); +} + +int +main (int argc, + char *argv[]) +{ + char *argv1; + + g_return_val_if_fail (argc > 1, 1); + argv1 = argv[1]; + + if (argc > 2) + memmove (&argv[1], &argv[2], (argc - 2) * sizeof (char *)); + + argc -= 1; + argv[argc] = NULL; + + g_test_init (&argc, &argv, NULL); + g_test_set_nonfatal_assertions (); + + if (g_strcmp0 (argv1, "pass") == 0) + { + g_test_add_func ("/pass", test_pass); + } + else if (g_strcmp0 (argv1, "skip") == 0) + { + g_test_add_func ("/skip", test_skip); + } + else if (g_strcmp0 (argv1, "incomplete") == 0) + { + g_test_add_func ("/incomplete", test_incomplete); + } + else if (g_strcmp0 (argv1, "fail") == 0) + { + g_test_add_func ("/fail", test_fail); + } + else if (g_strcmp0 (argv1, "all-non-failures") == 0) + { + g_test_add_func ("/pass", test_pass); + g_test_add_func ("/skip", test_skip); + g_test_add_func ("/incomplete", test_incomplete); + } + else if (g_strcmp0 (argv1, "all") == 0) + { + g_test_add_func ("/pass", test_pass); + g_test_add_func ("/skip", test_skip); + g_test_add_func ("/incomplete", test_incomplete); + g_test_add_func ("/fail", test_fail); + } + else + { + g_assert_not_reached (); + } + + return g_test_run (); +} diff --git a/glib/tests/testing.c b/glib/tests/testing.c index 64c5244f9..716516e2c 100644 --- a/glib/tests/testing.c +++ b/glib/tests/testing.c @@ -20,6 +20,8 @@ * if advised of the possibility of such damage. */ +#include "config.h" + /* We want to distinguish between messages originating from libglib * and messages originating from this program. */ @@ -890,6 +892,128 @@ test_combining (void) g_ptr_array_unref (argv); } +/* Test the TAP output when a test suite is run with --tap. */ +static void +test_tap (void) +{ + const char *testing_helper; + GPtrArray *argv; + GError *error = NULL; + int status; + gchar *output; + + testing_helper = g_test_get_filename (G_TEST_BUILT, "testing-helper" EXEEXT, NULL); + + g_test_message ("pass"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "pass"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_exit_status (status, &error); + g_assert_no_error (error); + g_assert_nonnull (strstr (output, "\nok 1 /pass\n")); + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("skip"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_exit_status (status, &error); + g_assert_no_error (error); + g_assert_nonnull (strstr (output, "\nok 1 /skip # SKIP not enough tea\n")); + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("incomplete"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "incomplete"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_exit_status (status, &error); + g_assert_no_error (error); + g_assert_nonnull (strstr (output, "\nnot ok 1 /incomplete # TODO mind reading not implemented yet\n")); + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("fail"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "fail"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_exit_status (status, &error); + g_assert_error (error, G_SPAWN_EXIT_ERROR, 1); + g_assert_nonnull (strstr (output, "\nnot ok 1 /fail\n")); + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("all"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "all"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, NULL, NULL, &status, + &error); + g_assert_error (error, G_SPAWN_EXIT_ERROR, 1); + g_clear_error (&error); + g_ptr_array_unref (argv); + + g_test_message ("all-non-failures"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "all-non-failures"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, NULL, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_exit_status (status, &error); + g_assert_no_error (error); + + g_ptr_array_unref (argv); +} + int main (int argc, char *argv[]) @@ -964,5 +1088,7 @@ main (int argc, g_test_add_func ("/misc/incomplete", test_incomplete); g_test_add_func ("/misc/timeout", test_subprocess_timed_out); + g_test_add_func ("/tap", test_tap); + return g_test_run(); }