From 7058efb39086101bc42e676be8b929e5724b0505 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 9 Jan 2019 10:47:24 +0000 Subject: [PATCH 1/2] =?UTF-8?q?ci:=20Don=E2=80=99t=20run=20tests=20tagged?= =?UTF-8?q?=20as=20=E2=80=98flaky=E2=80=99=20on=20the=20CI=20machines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This effectively renders those tests useless (since realistically nobody runs tests locally), but it’s better than every other CI run failing for unrelated reasons. The idea is that the ‘flaky’ tag can be temporarily applied to a test while a problem is being investigated or fixed, and then removed later. Signed-off-by: Philip Withnall --- .gitlab-ci.yml | 4 ++-- .gitlab-ci/test-msvc.bat | 2 +- .gitlab-ci/test-msys2.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bb16a9876..52bf81903 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,7 +34,7 @@ fedora-x86_64: - ninja -C _build - mkdir -p _coverage - lcov --config-file .gitlab-ci/lcovrc --directory _build --capture --initial --output-file "_coverage/${CI_JOB_NAME}-baseline.lcov" - - meson test -C _build --timeout-multiplier ${MESON_TEST_TIMEOUT_MULTIPLIER} + - meson test -C _build --timeout-multiplier ${MESON_TEST_TIMEOUT_MULTIPLIER} --no-suite flaky - lcov --config-file .gitlab-ci/lcovrc --directory _build --capture --output-file "_coverage/${CI_JOB_NAME}.lcov" # FIXME: We should run all installed tests, but do only this one for now # because it cannot run uninstalled. Reconfigure with dtrace disabled @@ -169,7 +169,7 @@ freebsd-11-x86_64: # FIXME: extattr(2) support: https://gitlab.gnome.org/GNOME/glib/issues/1404 - meson ${MESON_COMMON_OPTIONS} -Db_lundef=false -Diconv=gnu -Dxattr=false _build - ninja -C _build - - meson test -C _build --timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" + - meson test -C _build --timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" --no-suite flaky except: - tags artifacts: diff --git a/.gitlab-ci/test-msvc.bat b/.gitlab-ci/test-msvc.bat index c3b19cd34..e5f807387 100644 --- a/.gitlab-ci/test-msvc.bat +++ b/.gitlab-ci/test-msvc.bat @@ -10,7 +10,7 @@ meson _build || goto :error ninja -C _build || goto :error :: FIXME: dont ignore test errors -meson test -C _build --timeout-multiplier %MESON_TEST_TIMEOUT_MULTIPLIER% +meson test -C _build --timeout-multiplier %MESON_TEST_TIMEOUT_MULTIPLIER% --no-suite flaky :: FIXME: can we get code coverage support? diff --git a/.gitlab-ci/test-msys2.sh b/.gitlab-ci/test-msys2.sh index ed129c038..4cb710857 100755 --- a/.gitlab-ci/test-msys2.sh +++ b/.gitlab-ci/test-msys2.sh @@ -51,7 +51,7 @@ ninja --output-file "${DIR}/_coverage/${CI_JOB_NAME}-baseline.lcov" # FIXME: fix the test suite -meson test --timeout-multiplier ${MESON_TEST_TIMEOUT_MULTIPLIER} || true +meson test --timeout-multiplier ${MESON_TEST_TIMEOUT_MULTIPLIER} --no-suite flaky || true "${LCOV}" \ --quiet \ From 956a54d6d6b08c184a8b71faf71685b93f19e350 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 9 Jan 2019 10:48:42 +0000 Subject: [PATCH 2/2] =?UTF-8?q?tests:=20Tag=20several=20rogue=20GIO=20test?= =?UTF-8?q?s=20as=20=E2=80=98flaky=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These have been flaky for a while now, and nobody has found the time to fix them properly. They’ve been disrupting development for altogether far too long. They need investigating in the following issues, fixing properly, and the ‘flaky’ tag removing from each one: • https://gitlab.gnome.org/GNOME/glib/issues/1644 • https://gitlab.gnome.org/GNOME/glib/issues/1634 • https://gitlab.gnome.org/GNOME/glib/issues/1614 • https://gitlab.gnome.org/GNOME/glib/issues/1515 Signed-off-by: Philip Withnall --- gio/tests/meson.build | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gio/tests/meson.build b/gio/tests/meson.build index 5bbc07176..e969d9246 100644 --- a/gio/tests/meson.build +++ b/gio/tests/meson.build @@ -55,7 +55,7 @@ gio_tests = { 'memory-output-stream' : {}, 'monitor' : {}, 'mount-operation' : {}, - 'network-address' : {'extra_sources': ['mock-resolver.c']}, + 'network-address' : {'extra_sources': ['mock-resolver.c'], 'suite': ['flaky']}, 'network-monitor' : {}, 'network-monitor-race' : {}, 'permission' : {}, @@ -73,7 +73,7 @@ gio_tests = { 'vfs' : {}, 'volumemonitor' : {}, 'glistmodel' : {}, - 'testfilemonitor' : {'suite' : ['slow']}, + 'testfilemonitor' : {'suite' : ['slow', 'flaky']}, 'thumbnail-verification' : {}, 'tls-certificate' : {'extra_sources' : ['gtesttlsbackend.c']}, 'tls-interaction' : {'extra_sources' : ['gtesttlsbackend.c']}, @@ -236,10 +236,10 @@ if host_machine.system() != 'windows' 'extra_sources' : extra_sources, 'suite' : ['slow'], }, - 'gdbus-auth' : {'extra_sources' : extra_sources}, - 'gdbus-bz627724' : {'extra_sources' : extra_sources}, + 'gdbus-auth' : {'extra_sources' : extra_sources, 'suite': ['flaky']}, + 'gdbus-bz627724' : {'extra_sources' : extra_sources, 'suite': ['flaky']}, 'gdbus-close-pending' : {'extra_sources' : extra_sources}, - 'gdbus-connection' : {'extra_sources' : extra_sources}, + 'gdbus-connection' : {'extra_sources' : extra_sources, 'suite': ['flaky']}, 'gdbus-connection-loss' : {'extra_sources' : extra_sources}, 'gdbus-connection-slow' : {'extra_sources' : extra_sources}, 'gdbus-error' : {'extra_sources' : extra_sources}, @@ -262,7 +262,7 @@ if host_machine.system() != 'windows' }, 'gdbus-threading' : { 'extra_sources' : extra_sources, - 'suite' : ['slow'], + 'suite' : ['slow', 'flaky'], }, 'gmenumodel' : { 'extra_sources' : extra_sources,