tests: Stop ignoring test failures on macOS

Test failures were previously ignored on macOS because there are 12
tests which consistently fail (and have not yet been fixed, because
there are no regularly active macOS maintainers for GLib; you could help
here!).

However, this means that new test failures can’t be spotted.

So, explicitly mark those 12 tests as `should_fail` on macOS, and then
make other test failures cause failure of the CI run.

We can track the process of fixing those 12 tests on #1392 and #1251.

Signed-off-by: Philip Withnall <pwithnall@endlessos.org>

Helps: #1392

.gitlab-ci.yml

@@ -449,8 +449,7 @@ macos:
             --werror
             _build
     - ninja -C _build
-    # FIXME: Multiple unit tests currently fails
+    - .gitlab-ci/run-tests.sh
-    - .gitlab-ci/run-tests.sh || true
   artifacts:
     reports:
       junit: "_build/${CI_JOB_NAME}-report.xml"

gio/tests/meson.build

@@ -42,14 +42,20 @@ giotypefuncs_inc = custom_target(
 
 #  Test programs buildable on all platforms
 gio_tests = {
-  'appmonitor' : {},
+  'appmonitor' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'async-close-output-stream' : {},
   'async-splice-output-stream' : {},
   'buffered-input-stream' : {},
   'buffered-output-stream' : {},
   'cancellable' : {},
   'contexts' : {},
-  'contenttype' : {},
+  'contenttype' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392 / https://gitlab.gnome.org/GNOME/glib/-/issues/1251
+    'should_fail' : host_system == 'darwin',
+  },
   'converter-stream' : {},
   'credentials' : {},
   'cxx' : {
@@ -59,10 +65,19 @@ gio_tests = {
   'data-output-stream' : {},
   'fileattributematcher' : {},
   'filter-streams' : {},
-  'giomodule' : {},
+  'giomodule' : {
-  'gsubprocess' : {},
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
+  'gsubprocess' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'g-file' : {},
-  'g-file-info' : {},
+  'g-file-info' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'g-icon' : {},
   'gdbus-addresses' : {},
   'gdbus-message' : {},
@@ -72,7 +87,11 @@ gio_tests = {
   'memory-monitor' : {},
   'memory-output-stream' : {},
   'mount-operation' : {},
-  'network-address' : {'extra_sources': ['mock-resolver.c']},
+  'network-address' : {
+    'extra_sources': ['mock-resolver.c'],
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'network-monitor' : {},
   'network-monitor-race' : {},
   'permission' : {},
@@ -83,7 +102,10 @@ gio_tests = {
   'simple-async-result' : {},
   'simple-proxy' : {},
   'sleepy-stream' : {},
-  'socket' : {},
+  'socket' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'socket-listener' : {},
   'socket-service' : {},
   'srvtarget' : {},
@@ -97,7 +119,10 @@ gio_tests = {
   'tls-interaction' : {'extra_sources' : ['gtesttlsbackend.c']},
   'tls-database' : {'extra_sources' : ['gtesttlsbackend.c']},
   'tls-bindings' : {'extra_sources' : ['gtesttlsbackend.c']},
-  'gdbus-address-get-session' : {},
+  'gdbus-address-get-session' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'win32-appinfo' : {},
 }
 
@@ -156,7 +181,9 @@ if host_machine.system() != 'windows'
     'file' : {},
     'gdbus-peer' : {
       'dependencies' : [libgdbus_example_objectmanager_dep],
-      'install_rpath' : installed_tests_execdir
+      'install_rpath' : installed_tests_execdir,
+      # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+      'should_fail' : host_system == 'darwin',
     },
     'gdbus-peer-object-manager' : {},
     'live-g-file' : {},
@@ -779,6 +806,8 @@ if not meson.is_cross_build() or meson.has_exe_wrapper()
   gio_tests += {
     'resources' : {
       'extra_sources' : resources_extra_sources,
+      # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+      'should_fail' : host_system == 'darwin',
     },
   }
 endif
@@ -832,6 +861,7 @@ foreach test_name, extra_args : gio_tests
     suite : suite,
     is_parallel : extra_args.get('is_parallel', true),
     depends : extra_args.get('depends', []),
+    should_fail : extra_args.get('should_fail', false),
   )
 endforeach
 

glib/tests/meson.build

@@ -19,9 +19,15 @@ glib_tests = {
     'source' : ['cxx.cpp'],
   },
   'dataset' : {},
-  'date' : {},
+  'date' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'dir' : {},
-  'environment' : {},
+  'environment' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'error' : {},
   'fileutils' : {},
   'gdatetime' : {
@@ -241,7 +247,12 @@ foreach test_name, extra_args : glib_tests
 
   suite = ['glib'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
-  test(test_name, exe, env : test_env, timeout : timeout, suite : suite)
+  test(test_name, exe,
+    env : test_env,
+    timeout : timeout,
+    suite : suite,
+    should_fail : extra_args.get('should_fail', false),
+  )
 endforeach
 
 executable('spawn-path-search-helper', 'spawn-path-search-helper.c',