testutils: Use prctl PR_SET_DUMPABLE to silence core dumps on Linux

Otherwise, crashing tests like assert-msg-test will still report to
pipe-based crash reporting frameworks like systemd-coredump, even though
the RLIMIT_CORE limit is zero.

Signed-off-by: Simon McVittie <smcv@collabora.com>

glib/gtestutils.c

@@ -34,6 +34,9 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <inttypes.h>
+#ifdef HAVE_SYS_PRCTL_H
+#include <sys/prctl.h>
+#endif
 #ifdef HAVE_SYS_RESOURCE_H
 #include <sys/resource.h>
 #endif
@@ -1321,6 +1324,12 @@ g_test_disable_crash_reporting (void)
 
   (void) setrlimit (RLIMIT_CORE, &limit);
 #endif
+
+#if defined(HAVE_PRCTL) && defined(PR_SET_DUMPABLE)
+  /* On Linux, RLIMIT_CORE = 0 is ignored if core dumps are
+   * configured to be written to a pipe, but PR_SET_DUMPABLE is not. */
+  (void) prctl (PR_SET_DUMPABLE, 0, 0, 0, 0);
+#endif
 }
 
 /* We intentionally parse the command line without GOptionContext

meson.build

@@ -381,6 +381,7 @@ headers = [
   'sys/mnttab.h',
   'sys/mount.h',
   'sys/param.h',
+  'sys/prctl.h',
   'sys/resource.h',
   'sys/select.h',
   'sys/statfs.h',
@@ -695,6 +696,11 @@ if glib_conf.has('HAVE_SYS_STATFS_H') or glib_conf.has('HAVE_SYS_MOUNT_H')
 else
   have_func_statfs = false
 endif
+if glib_conf.has('HAVE_SYS_PRCTL_H')
+  functions += ['prctl']
+else
+  have_func_prctl = false
+endif
 
 if host_system == 'windows'
   iphlpapi_dep = cc.find_library('iphlpapi')