From fac6402e7b8dfe40e97945e77c8d2d02699cca8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Sat, 17 Dec 2022 05:33:57 +0100 Subject: [PATCH] gtestutils: Use TAP version 14 Subtests syntax to show subtests output Instead of just commenting all the output of sub-processes we can just use the TAP 14 syntax for subtests, by using 4-spaces to indent the subtests output. This may not work perfectly when there are sub-process that may write output mixed with the parent, but it should be enough to expose the hierarchy. --- glib/gtestutils.c | 39 ++- glib/tests/testing.c | 729 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 747 insertions(+), 21 deletions(-) diff --git a/glib/gtestutils.c b/glib/gtestutils.c index 24c109f7a..000ec50de 100644 --- a/glib/gtestutils.c +++ b/glib/gtestutils.c @@ -58,6 +58,7 @@ #include "glib-private.h" #include "gutilsprivate.h" +#define TAP_SUBTEST_PREFIX " " /* a 4-space indented line */ /** * SECTION:testing @@ -994,16 +995,23 @@ g_test_log (GTestLogType lbit, gchar *astrings[3] = { NULL, NULL, NULL }; guint8 *dbuffer; guint32 dbufferlen; + gboolean is_subtest; + const char *tap_prefix; + + is_subtest = test_is_subtest || test_in_forked_child || test_in_subprocess; + tap_prefix = test_tap_log && is_subtest ? TAP_SUBTEST_PREFIX : ""; switch (lbit) { case G_TEST_LOG_START_BINARY: if (test_tap_log) { - if (!test_is_subtest && !test_in_subprocess && !test_in_forked_child) - g_print ("TAP version 13\n"); + if (!is_subtest) + g_print ("TAP version 14\n"); + else + g_print ("# Subtest: %s\n", test_argv0); - g_print ("# random seed: %s\n", string2); + g_print ("%s# random seed: %s\n", tap_prefix, string2); } else if (g_test_verbose ()) { @@ -1016,9 +1024,9 @@ g_test_log (GTestLogType lbit, /* We only print the TAP "plan" (1..n) ahead of time if we did * not use the -p option to select specific tests to be run. */ if (string1[0] != 0) - g_print ("# Start of %s tests\n", string1); + g_print ("%s# Start of %s tests\n", tap_prefix, string1); else if (test_paths == NULL) - g_print ("%s1..%d\n", test_is_subtest ? "# " : "", test_count); + g_print ("%s1..%d\n", tap_prefix, test_count); } break; case G_TEST_LOG_STOP_SUITE: @@ -1028,9 +1036,9 @@ g_test_log (GTestLogType lbit, * we were using -p, we need to print how many tests we ran at * the end instead. */ if (string1[0] != 0) - g_print ("# End of %s tests\n", string1); + g_print ("%s# End of %s tests\n", tap_prefix, string1); else if (test_paths != NULL) - g_print ("%s1..%d\n", test_is_subtest ? "# " : "", test_run_count); + g_print ("%s1..%d\n", tap_prefix, test_run_count); } break; case G_TEST_LOG_STOP_CASE: @@ -1051,8 +1059,8 @@ g_test_log (GTestLogType lbit, else tap_output = g_string_new ("ok"); - if (test_is_subtest) - g_string_prepend (tap_output, "# "); + if (is_subtest) + g_string_prepend (tap_output, TAP_SUBTEST_PREFIX); g_string_append_printf (tap_output, " %d %s", test_run_count, string1); if (result == G_TEST_RUN_INCOMPLETE) @@ -1072,7 +1080,7 @@ g_test_log (GTestLogType lbit, if (fail && test_mode_fatal) { if (test_tap_log) - g_print ("%sBail out!\n", test_is_subtest ? "# " : ""); + g_print ("%sBail out!\n", tap_prefix); g_abort (); } if (result == G_TEST_RUN_SKIPPED || result == G_TEST_RUN_INCOMPLETE) @@ -1081,19 +1089,19 @@ g_test_log (GTestLogType lbit, case G_TEST_LOG_SKIP_CASE: if (test_tap_log) { - g_print ("%sok %d %s # SKIP\n", test_is_subtest ? "# " : "", + g_print ("%sok %d %s # SKIP\n", tap_prefix, test_run_count, string1); } break; case G_TEST_LOG_MIN_RESULT: if (test_tap_log) - g_print ("# min perf: %s\n", string1); + g_print ("%s# min perf: %s\n", tap_prefix, string1); else if (g_test_verbose ()) g_print ("(MINPERF:%s)\n", string1); break; case G_TEST_LOG_MAX_RESULT: if (test_tap_log) - g_print ("# max perf: %s\n", string1); + g_print ("%s# max perf: %s\n", tap_prefix, string1); else if (g_test_verbose ()) g_print ("(MAXPERF:%s)\n", string1); break; @@ -1101,7 +1109,7 @@ g_test_log (GTestLogType lbit, if (test_tap_log) { if (strchr (string1, '\n') == NULL) - g_print ("# %s\n", string1); + g_print ("%s# %s\n", tap_prefix, string1); else { GString *output = g_string_new (NULL); @@ -1110,6 +1118,7 @@ g_test_log (GTestLogType lbit, do { const char *next = strchr (line, '\n'); + g_string_append (output, tap_prefix); g_string_append (output, "# "); if (next) @@ -1135,7 +1144,7 @@ g_test_log (GTestLogType lbit, break; case G_TEST_LOG_ERROR: if (test_tap_log) - g_print ("%sBail out! %s\n", test_is_subtest ? "# " : "", string1); + g_print ("%sBail out! %s\n", tap_prefix, string1); else if (g_test_verbose ()) g_print ("(ERROR: %s)\n", string1); break; diff --git a/glib/tests/testing.c b/glib/tests/testing.c index c97f9db62..53946f2c6 100644 --- a/glib/tests/testing.c +++ b/glib/tests/testing.c @@ -35,7 +35,8 @@ #include #include -#define TAP_VERSION G_STRINGIFY (13) +#define TAP_VERSION G_STRINGIFY (14) +#define TAP_SUBTEST_PREFIX " " /* test assertion variants */ static void @@ -1116,6 +1117,7 @@ test_tap (void) g_spawn_check_wait_status (status, &error); g_assert_no_error (error); g_assert_true (g_str_has_prefix (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "# Subtest: ")); g_assert_nonnull (strstr (output, "\nok 1 /pass\n")); g_free (output); g_ptr_array_unref (argv); @@ -1136,6 +1138,7 @@ test_tap (void) g_spawn_check_wait_status (status, &error); g_assert_no_error (error); g_assert_true (g_str_has_prefix (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "# Subtest: ")); g_assert_nonnull (strstr (output, "\nok 1 /skip # SKIP not enough tea\n")); g_free (output); g_ptr_array_unref (argv); @@ -1156,6 +1159,7 @@ test_tap (void) g_spawn_check_wait_status (status, &error); g_assert_no_error (error); g_assert_true (g_str_has_prefix (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "# Subtest: ")); g_assert_nonnull (strstr (output, "\nok 1 /skip-printf # SKIP not enough coffee\n")); g_free (output); g_ptr_array_unref (argv); @@ -1176,6 +1180,7 @@ test_tap (void) g_spawn_check_wait_status (status, &error); g_assert_no_error (error); g_assert_true (g_str_has_prefix (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "# Subtest: ")); g_assert_nonnull (strstr (output, "\nnot ok 1 /incomplete # TODO mind reading not implemented yet\n")); g_free (output); g_ptr_array_unref (argv); @@ -1196,6 +1201,7 @@ test_tap (void) g_spawn_check_wait_status (status, &error); g_assert_no_error (error); g_assert_true (g_str_has_prefix (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "# Subtest: ")); g_assert_nonnull (strstr (output, "\nnot ok 1 /incomplete-printf # TODO telekinesis not implemented yet\n")); g_free (output); g_ptr_array_unref (argv); @@ -1216,6 +1222,7 @@ test_tap (void) g_spawn_check_wait_status (status, &error); g_assert_error (error, G_SPAWN_EXIT_ERROR, 1); g_assert_true (g_str_has_prefix (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "# Subtest: ")); g_assert_nonnull (strstr (output, "\nnot ok 1 /fail\n")); g_free (output); g_clear_error (&error); @@ -1237,6 +1244,7 @@ test_tap (void) g_spawn_check_wait_status (status, &error); g_assert_error (error, G_SPAWN_EXIT_ERROR, 1); g_assert_true (g_str_has_prefix (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "# Subtest: ")); g_assert_nonnull (strstr (output, "\nnot ok 1 /fail-printf - this test intentionally left failing\n")); g_free (output); g_clear_error (&error); @@ -1293,6 +1301,7 @@ test_tap (void) &error); g_assert_no_error (error); g_assert_true (g_str_has_prefix (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "# Subtest: ")); g_assert_nonnull (strstr (output, "1..10\n")); g_assert_nonnull (strstr (output, "\nok 1 /a # SKIP\n")); g_assert_nonnull (strstr (output, "\nok 2 /b # SKIP\n")); @@ -1326,6 +1335,7 @@ test_tap (void) &error); g_assert_no_error (error); g_assert_true (g_str_has_prefix (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "# Subtest: ")); g_assert_nonnull (strstr (output, "1..10\n")); g_assert_nonnull (strstr (output, "\nok 1 /a\n")); g_assert_nonnull (strstr (output, "\nok 2 /b\n")); @@ -1359,6 +1369,7 @@ test_tap (void) &error); g_assert_no_error (error); g_assert_true (g_str_has_prefix (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "# Subtest: ")); g_assert_nonnull (strstr (output, "1..10\n")); g_assert_nonnull (strstr (output, "\nok 1 /a # SKIP\n")); g_assert_nonnull (strstr (output, "\nok 2 /b # SKIP\n")); @@ -1396,6 +1407,7 @@ test_tap (void) &error); g_assert_no_error (error); g_assert_true (g_str_has_prefix (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "# Subtest: ")); g_assert_nonnull (strstr (output, "\nok 1 /c/a\n")); g_assert_nonnull (strstr (output, "\nok 2 /c/a\n")); g_assert_nonnull (strstr (output, "\nok 3 /b\n")); @@ -1428,6 +1440,7 @@ test_tap (void) &error); g_assert_no_error (error); g_assert_true (g_str_has_prefix (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "# Subtest: ")); g_assert_nonnull (strstr (output, "\nok 1 /c/a\n")); g_assert_nonnull (strstr (output, "\nok 2 /c/a\n")); g_assert_nonnull (strstr (output, "\nok 3 /b\n")); @@ -1459,6 +1472,7 @@ test_tap (void) &error); g_assert_no_error (error); g_assert_true (g_str_has_prefix (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "# Subtest: ")); g_assert_nonnull (strstr (output, "\nok 1 /b/b\n")); g_assert_nonnull (strstr (output, "\nok 2 /b/b/a\n")); g_assert_nonnull (strstr (output, "\n1..2\n")); @@ -1517,6 +1531,7 @@ test_tap (void) &error); g_assert_no_error (error); g_assert_true (g_str_has_prefix (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "# Subtest: ")); g_assert_nonnull (strstr (output, "1..10\n")); g_assert_nonnull (strstr (output, "\nok 1 /a # SKIP by request")); g_assert_nonnull (strstr (output, "\nok 2 /b # SKIP by request")); @@ -1556,6 +1571,7 @@ test_tap (void) &error); g_assert_no_error (error); g_assert_true (g_str_has_prefix (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "# Subtest: ")); g_assert_nonnull (strstr (output, "1..10\n")); g_assert_nonnull (strstr (output, "\nok 1 /a # SKIP by request")); g_assert_nonnull (strstr (output, "\nok 2 /b\n")); @@ -1604,8 +1620,636 @@ test_tap (void) g_strfreev (envp); } +/* Test the TAP output when a test suite is run with --tap. */ +static void +test_tap_subtest (void) +{ + const char *testing_helper; + GPtrArray *argv; + GError *error = NULL; + int status; + gchar *output; + char** envp = NULL; + + testing_helper = g_test_get_filename (G_TEST_BUILT, "testing-helper" EXEEXT, NULL); + + g_test_message ("pass"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "pass"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + envp = g_get_environ (); + g_assert_nonnull (g_environ_getenv (envp, "G_TEST_ROOT_PROCESS")); + g_clear_pointer (&envp, g_strfreev); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_true (g_str_has_prefix (output, "# Subtest: ")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 1 /pass\n")); + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("skip"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_true (g_str_has_prefix (output, "# Subtest: ")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 1 /skip # SKIP not enough tea\n")); + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("skip with printf format"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip-printf"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_true (g_str_has_prefix (output, "# Subtest: ")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 1 /skip-printf # SKIP not enough coffee\n")); + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("incomplete"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "incomplete"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_null (strstr (output, "\n# Subtest: ")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "not ok 1 /incomplete # TODO mind reading not implemented yet\n")); + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("incomplete with printf format"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "incomplete-printf"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_null( strstr (output, "\n# Subtest: ")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "not ok 1 /incomplete-printf # TODO telekinesis not implemented yet\n")); + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("fail"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "fail"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_wait_status (status, &error); + g_assert_error (error, G_SPAWN_EXIT_ERROR, 1); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_null( strstr (output, "\n# Subtest: ")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "not ok 1 /fail\n")); + g_free (output); + g_clear_error (&error); + g_ptr_array_unref (argv); + + g_test_message ("fail with message"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "fail-printf"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_wait_status (status, &error); + g_assert_error (error, G_SPAWN_EXIT_ERROR, 1); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_null( strstr (output, "\n# Subtest: ")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "not ok 1 /fail-printf - this test intentionally left failing\n")); + g_free (output); + g_clear_error (&error); + g_ptr_array_unref (argv); + + g_test_message ("all"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "all"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, NULL, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_wait_status (status, &error); + g_assert_error (error, G_SPAWN_EXIT_ERROR, 1); + g_clear_error (&error); + g_ptr_array_unref (argv); + + g_test_message ("all-non-failures"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "all-non-failures"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, NULL, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + + g_ptr_array_unref (argv); + + g_test_message ("--GTestSkipCount"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip-options"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, "--GTestSkipCount"); + g_ptr_array_add (argv, "2"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_null( strstr (output, "\n# Subtest: ")); + g_assert_nonnull (strstr (output, TAP_SUBTEST_PREFIX "1..10\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 1 /a # SKIP\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 2 /b # SKIP\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 3 /b/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 4 /b/b\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 5 /b/b/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 6 /prefix/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 7 /prefix/b/b\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 8 /prefix-long/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 9 /c/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 10 /d/a\n")); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("--GTestSkipCount=0 is the same as omitting it"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip-options"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, "--GTestSkipCount"); + g_ptr_array_add (argv, "0"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_null( strstr (output, "\n# Subtest: ")); + g_assert_nonnull (strstr (output, TAP_SUBTEST_PREFIX "1..10\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 1 /a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 2 /b\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 3 /b/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 4 /b/b\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 5 /b/b/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 6 /prefix/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 7 /prefix/b/b\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 8 /prefix-long/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 9 /c/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 10 /d/a\n")); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("--GTestSkipCount > number of tests skips all"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip-options"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, "--GTestSkipCount"); + g_ptr_array_add (argv, "11"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_null( strstr (output, "\n# Subtest: ")); + g_assert_nonnull (strstr (output, TAP_SUBTEST_PREFIX "1..10\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 1 /a # SKIP\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 2 /b # SKIP\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 3 /b/a # SKIP\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 4 /b/b # SKIP\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 5 /b/b/a # SKIP\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 6 /prefix/a # SKIP\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 7 /prefix/b/b # SKIP\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 8 /prefix-long/a # SKIP\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 9 /c/a # SKIP\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 10 /d/a # SKIP\n")); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("-p"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip-options"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/c/a"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/c/a"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/b"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_null( strstr (output, "\n# Subtest: ")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 1 /c/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 2 /c/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 3 /b\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 4 /b/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 5 /b/b\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "1..5\n")); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("--run-prefix"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip-options"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, "-r"); + g_ptr_array_add (argv, "/c/a"); + g_ptr_array_add (argv, "-r"); + g_ptr_array_add (argv, "/c/a"); + g_ptr_array_add (argv, "--run-prefix"); + g_ptr_array_add (argv, "/b"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_null( strstr (output, "\n# Subtest: ")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 1 /c/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 2 /c/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 3 /b\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 4 /b/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 5 /b/b\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 6 /b/b/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "1..6\n")); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("--run-prefix 2"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip-options"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, "-r"); + g_ptr_array_add (argv, "/pre"); + g_ptr_array_add (argv, "--run-prefix"); + g_ptr_array_add (argv, "/b/b"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_null( strstr (output, "\n# Subtest: ")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 1 /b/b\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 2 /b/b/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "1..2\n")); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("--run-prefix conflict"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip-options"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, "-r"); + g_ptr_array_add (argv, "/c/a"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/c/a"); + g_ptr_array_add (argv, "--run-prefix"); + g_ptr_array_add (argv, "/b"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + g_spawn_check_wait_status (status, &error); + g_assert_nonnull (error); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_null( strstr (output, "\n# Subtest: ")); + g_assert_nonnull (strstr (output, "do not mix [-r | --run-prefix] with '-p'\n")); + g_clear_error (&error); + + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("-s"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip-options"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, "-s"); + g_ptr_array_add (argv, "/a"); + g_ptr_array_add (argv, "-s"); + g_ptr_array_add (argv, "/b"); + g_ptr_array_add (argv, "-s"); + g_ptr_array_add (argv, "/pre"); + g_ptr_array_add (argv, "-s"); + g_ptr_array_add (argv, "/c/a"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_null( strstr (output, "\n# Subtest: ")); + g_assert_nonnull (strstr (output, TAP_SUBTEST_PREFIX "1..10\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 1 /a # SKIP by request")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 2 /b # SKIP by request")); + /* "-s /b" would skip a test named exactly /b, but not a test named + * /b/anything */ + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 3 /b/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 4 /b/b\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 5 /b/b/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 6 /prefix/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 7 /prefix/b/b\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 8 /prefix-long/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 9 /c/a # SKIP by request")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 10 /d/a\n")); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("--skip-prefix"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip-options"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, "-x"); + g_ptr_array_add (argv, "/a"); + g_ptr_array_add (argv, "--skip-prefix"); + g_ptr_array_add (argv, "/pre"); + g_ptr_array_add (argv, "-x"); + g_ptr_array_add (argv, "/c/a"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + g_assert_null (strstr (output, "TAP version " TAP_VERSION)); + g_assert_true (g_str_has_prefix (output, "# Subtest: ")); + g_assert_nonnull (strstr (output, TAP_SUBTEST_PREFIX "1..10\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 1 /a # SKIP by request")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 2 /b\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 3 /b/a\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 4 /b/b\n")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 5 /b/b/a\n")); + /* "--skip-prefix /pre" will skip all test path which begins with /pre */ + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 6 /prefix/a # SKIP by request")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 7 /prefix/b/b # SKIP by request")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 8 /prefix-long/a # SKIP by request")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 9 /c/a # SKIP by request")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX "ok 10 /d/a\n")); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + + g_free (output); + g_ptr_array_unref (argv); +} + static void test_tap_summary (void) +{ + const char *testing_helper; + GPtrArray *argv; + GError *error = NULL; + int status; + gchar *output; + char **envp; + + g_test_summary ("Test the output of g_test_summary() from the TAP output of a test."); + + testing_helper = g_test_get_filename (G_TEST_BUILT, "testing-helper" EXEEXT, NULL); + + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "summary"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + /* Remove the G_TEST_ROOT_PROCESS env so it will be considered a standalone test */ + envp = g_get_environ (); + g_assert_nonnull (g_environ_getenv (envp, "G_TEST_ROOT_PROCESS")); + envp = g_environ_unsetenv (g_steal_pointer (&envp), "G_TEST_ROOT_PROCESS"); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + g_assert_null (strstr (output, "# Subtest: ")); + + /* Note: The test path in the output is not `/tap/summary` because it’s the + * test path from testing-helper, not from this function. */g_assert_null (strstr (output, "# Subtest: ")); + g_assert_nonnull (strstr (output, "\n# /summary summary: Tests that g_test_summary() " + "works with TAP, by outputting a known " + "summary message in testing-helper, and " + "checking for it in the TAP output later.\n")); + g_free (output); + g_ptr_array_unref (argv); + g_strfreev (envp); +} + +static void +test_tap_subtest_summary (void) { const char *testing_helper; GPtrArray *argv; @@ -1633,10 +2277,13 @@ test_tap_summary (void) g_assert_no_error (error); /* Note: The test path in the output is not `/tap/summary` because it’s the * test path from testing-helper, not from this function. */ - g_assert_nonnull (strstr (output, "\n# /summary summary: Tests that g_test_summary() " - "works with TAP, by outputting a known " - "summary message in testing-helper, and " - "checking for it in the TAP output later.\n")); + g_assert_true (g_str_has_prefix (output, "# Subtest: ")); + g_assert_nonnull (strstr (output, + "\n" TAP_SUBTEST_PREFIX + "# /summary summary: Tests that g_test_summary() " + "works with TAP, by outputting a known " + "summary message in testing-helper, and " + "checking for it in the TAP output later.\n")); g_free (output); g_ptr_array_unref (argv); } @@ -1650,6 +2297,7 @@ test_tap_message (void) int status; gchar *output; char **output_lines; + char **envp; g_test_summary ("Test the output of g_test_message() from the TAP output of a test."); @@ -1661,7 +2309,12 @@ test_tap_message (void) g_ptr_array_add (argv, "--tap"); g_ptr_array_add (argv, NULL); - g_spawn_sync (NULL, (char **) argv->pdata, NULL, + /* Remove the G_TEST_ROOT_PROCESS env so it will be considered a standalone test */ + envp = g_get_environ (); + g_assert_nonnull (g_environ_getenv (envp, "G_TEST_ROOT_PROCESS")); + envp = g_environ_unsetenv (g_steal_pointer (&envp), "G_TEST_ROOT_PROCESS"); + + g_spawn_sync (NULL, (char **) argv->pdata, envp, G_SPAWN_STDERR_TO_DEV_NULL, NULL, NULL, &output, NULL, &status, &error); @@ -1670,6 +2323,8 @@ test_tap_message (void) g_spawn_check_wait_status (status, &error); g_assert_no_error (error); + g_assert_null (strstr (output, "# Subtest: ")); + const char *expected_tap_header = "\n1..1\n"; const char *interesting_lines = strstr (output, expected_tap_header); g_assert_nonnull (interesting_lines); @@ -1692,6 +2347,65 @@ test_tap_message (void) g_assert_cmpstr (output_lines[i++], ==, "# works with leading and trailing too"); g_assert_cmpstr (output_lines[i++], ==, "# "); + g_free (output); + g_strfreev (output_lines); + g_strfreev (envp); + g_ptr_array_unref (argv); +} + +static void +test_tap_subtest_message (void) +{ + const char *testing_helper; + GPtrArray *argv; + GError *error = NULL; + int status; + gchar *output; + char **output_lines; + + g_test_summary ("Test the output of g_test_message() from the TAP output of a sub-test."); + + testing_helper = g_test_get_filename (G_TEST_BUILT, "testing-helper" EXEEXT, NULL); + + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "message"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + + g_spawn_check_wait_status (status, &error); + g_assert_no_error (error); + + g_assert_true (g_str_has_prefix (output, "# Subtest: ")); + + const char *expected_tap_header = "\n" TAP_SUBTEST_PREFIX "1..1\n"; + const char *interesting_lines = strstr (output, expected_tap_header); + g_assert_nonnull (interesting_lines); + interesting_lines += strlen (expected_tap_header); + + output_lines = g_strsplit (interesting_lines, "\n", -1); + g_assert_cmpuint (g_strv_length (output_lines), >=, 12); + + guint i = 0; + g_assert_cmpstr (output_lines[i++], ==, TAP_SUBTEST_PREFIX "# Tests that single line message works"); + g_assert_cmpstr (output_lines[i++], ==, TAP_SUBTEST_PREFIX "# Tests that multi"); + g_assert_cmpstr (output_lines[i++], ==, TAP_SUBTEST_PREFIX "# "); + g_assert_cmpstr (output_lines[i++], ==, TAP_SUBTEST_PREFIX "# line"); + g_assert_cmpstr (output_lines[i++], ==, TAP_SUBTEST_PREFIX "# message"); + g_assert_cmpstr (output_lines[i++], ==, TAP_SUBTEST_PREFIX "# works"); + g_assert_cmpstr (output_lines[i++], ==, TAP_SUBTEST_PREFIX "# "); + g_assert_cmpstr (output_lines[i++], ==, TAP_SUBTEST_PREFIX "# Tests that multi"); + g_assert_cmpstr (output_lines[i++], ==, TAP_SUBTEST_PREFIX "# line"); + g_assert_cmpstr (output_lines[i++], ==, TAP_SUBTEST_PREFIX "# message"); + g_assert_cmpstr (output_lines[i++], ==, TAP_SUBTEST_PREFIX "# works with leading and trailing too"); + g_assert_cmpstr (output_lines[i++], ==, TAP_SUBTEST_PREFIX "# "); + g_free (output); g_strfreev (output_lines); g_ptr_array_unref (argv); @@ -1826,8 +2540,11 @@ main (int argc, g_test_add_func ("/misc/path/second", test_path_second); g_test_add_func ("/tap", test_tap); + g_test_add_func ("/tap/subtest", test_tap_subtest); g_test_add_func ("/tap/summary", test_tap_summary); + g_test_add_func ("/tap/subtest/summary", test_tap_subtest_summary); g_test_add_func ("/tap/message", test_tap_message); + g_test_add_func ("/tap/subtest/message", test_tap_subtest_message); g_test_add_func ("/init/no_argv0", test_init_no_argv0);