diff --git a/0001-README-document-impact-of-balance-quota-groups.patch b/0001-README-document-impact-of-balance-quota-groups.patch new file mode 100644 index 0000000..c99edf7 --- /dev/null +++ b/0001-README-document-impact-of-balance-quota-groups.patch @@ -0,0 +1,37 @@ +From 3e8ae32c961f0a3a6ba32881e88a49d5cbf656cd Mon Sep 17 00:00:00 2001 +From: Jeff Mahoney +Date: Fri, 11 Jan 2019 09:57:48 -0500 +Subject: README: document impact of balance + quota groups +References: fate#325529 jsc#SLE-3188 + +The combination of balance and quota groups as the number of references +to extents rises (either via snapshots, dedupe, or reflink copies) can +be extreme. + +The file system performs quota group accounting for every extent relocated +and the result can be the appearance of stalled system as the +transaction commit can take a very long time. + +Signed-off-by: Jeff Mahoney +--- + README.md | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/README.md b/README.md +index 07a8104..d59f3d6 100644 +--- a/README.md ++++ b/README.md +@@ -66,6 +66,10 @@ the same set of blocks is affected. + + The balance command uses filters to do the work in smaller batches. + ++The impact with quota groups enabled can be extreme. The balance operation ++performs quota group accounting for every extent being relocated, which can ++have the impact of stalling the file system for an extended period of time. ++ + __Expected result:__ If possible all the underused chunks are removed, the + value of `total` in output of `btrfs fi df /path` should be lower than before. + Check the logs. +-- +2.16.4 + diff --git a/0002-sysconfig-reduce-default-balance-usage-thresholds-fa.patch b/0002-sysconfig-reduce-default-balance-usage-thresholds-fa.patch new file mode 100644 index 0000000..823279d --- /dev/null +++ b/0002-sysconfig-reduce-default-balance-usage-thresholds-fa.patch @@ -0,0 +1,56 @@ +From 194fd3a02f686737ae7d266d72d35a3418a5903b Mon Sep 17 00:00:00 2001 +From: Jeff Mahoney +Date: Fri, 25 Jan 2019 13:29:20 -0500 +Subject: sysconfig: reduce default balance usage thresholds +References: fate#325529 jsc#SLE-3188 + +In the early days of btrfs, the only way to clean up empty block groups +was to perform a balance operation to coalesce block groups that were +mostly free and release the newly empty block groups. + +Since then, btrfs cleans up empty block groups automatically so the need +for regular aggressive balancing has been reduced substantially. Since +workload shifts could still produce ENOSPC issues, we still want to +perform a periodic balance but we can do it much less aggressively. +--- + sysconfig.btrfsmaintenance | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/sysconfig.btrfsmaintenance b/sysconfig.btrfsmaintenance +index 4088254..5a34c83 100644 +--- a/sysconfig.btrfsmaintenance ++++ b/sysconfig.btrfsmaintenance +@@ -52,18 +52,18 @@ BTRFS_BALANCE_PERIOD="weekly" + + ## Path: System/File systems/btrfs + ## Type: string +-## Default: "1 5 10 20 30 40 50" ++## Default: "5 10" + # + # The usage percent for balancing data block groups. + # + # Note: default values should not disturb normal work but may not reclaim + # enough block groups. If you observe that, add higher values but beware that + # this will increase IO load on the system. +-BTRFS_BALANCE_DUSAGE="1 5 10 20 30 40 50" ++BTRFS_BALANCE_DUSAGE="5 10" + + ## Path: System/File systems/btrfs + ## Type: string +-## Default: "1 5 10 20 30" ++## Default: "3" + # + # The usage percent for balancing metadata block groups. The values are also + # used in case the filesystem has mixed blockgroups. +@@ -71,7 +71,7 @@ BTRFS_BALANCE_DUSAGE="1 5 10 20 30 40 50" + # Note: default values should not disturb normal work but may not reclaim + # enough block groups. If you observe that, add higher values but beware that + # this will increase IO load on the system. +-BTRFS_BALANCE_MUSAGE="1 5 10 20 30" ++BTRFS_BALANCE_MUSAGE="3" + + ## Path: System/File systems/btrfs + ## Type: string +-- +2.16.4 + diff --git a/0003-sysconfig-document-using-systemd.time-7-for-frequenc.patch b/0003-sysconfig-document-using-systemd.time-7-for-frequenc.patch new file mode 100644 index 0000000..c90d557 --- /dev/null +++ b/0003-sysconfig-document-using-systemd.time-7-for-frequenc.patch @@ -0,0 +1,53 @@ +From 0302324b7a824741a39e3928dc7a8617485e03ce Mon Sep 17 00:00:00 2001 +From: Jeff Mahoney +Date: Fri, 25 Jan 2019 13:35:54 -0500 +Subject: sysconfig: document using systemd.time(7) for frequency of operations +References: fate#325529 jsc#SLE-3188 + +On systems using systemd timers, the calendar event format documented in +systemd.time(7) may be used to specify when to run balance, defrag, +scrub, or trim events. This commit documents that validity. +--- + sysconfig.btrfsmaintenance | 12 ++++++++++++ + 1 file changed, 12 insertions(+) + +diff --git a/sysconfig.btrfsmaintenance b/sysconfig.btrfsmaintenance +index 5a34c83..b35e2cf 100644 +--- a/sysconfig.btrfsmaintenance ++++ b/sysconfig.btrfsmaintenance +@@ -48,6 +48,10 @@ BTRFS_BALANCE_MOUNTPOINTS="/" + ## ServiceRestart: btrfsmaintenance-refresh + # + # Frequency of periodic balance. ++# ++# The frequency may be specified using one of the listed values or ++# in the format documented in the "Calendar Events" section of systemd.time(7), ++# if available. + BTRFS_BALANCE_PERIOD="weekly" + + ## Path: System/File systems/btrfs +@@ -89,6 +93,10 @@ BTRFS_SCRUB_MOUNTPOINTS="/" + ## ServiceRestart: btrfsmaintenance-refresh + # + # Frequency of periodic scrub. ++# ++# The frequency may be specified using one of the listed values or ++# in the format documented in the "Calendar Events" section of systemd.time(7), ++# if available. + BTRFS_SCRUB_PERIOD="monthly" + + ## Path: System/File systems/btrfs +@@ -115,6 +123,10 @@ BTRFS_SCRUB_READ_ONLY="false" + # Frequency of periodic trim. Off by default so it does not collide with + # fstrim.timer . If you do not use the timer, turn it on here. The recommended + # period is 'weekly'. ++# ++# The frequency may be specified using one of the listed values or ++# in the format documented in the "Calendar Events" section of systemd.time(7), ++# if available. + BTRFS_TRIM_PERIOD="none" + + ## Path: System/File systems/btrfs +-- +2.16.4 + diff --git a/0004-btrfsmaintenance-refresh-cron.sh-validate-periods-fo.patch b/0004-btrfsmaintenance-refresh-cron.sh-validate-periods-fo.patch new file mode 100644 index 0000000..76b5df5 --- /dev/null +++ b/0004-btrfsmaintenance-refresh-cron.sh-validate-periods-fo.patch @@ -0,0 +1,39 @@ +From 7851de9a30610885f7fc2f8238128f1c4f2d3593 Mon Sep 17 00:00:00 2001 +From: Jeff Mahoney +Date: Fri, 25 Jan 2019 14:08:13 -0500 +Subject: btrfsmaintenance-refresh-cron.sh: validate periods for cron-based systems +References: fate#325529 jsc#SLE-3188 + +Now that we accept systemd.time(7) calendar events, ensure that we don't +try to use them on cron-based systems. Instead of removing all existing +jobs silently, refuse to change the period and print an error. +--- + btrfsmaintenance-refresh-cron.sh | 12 ++++++++++++ + 1 file changed, 12 insertions(+) + +diff --git a/btrfsmaintenance-refresh-cron.sh b/btrfsmaintenance-refresh-cron.sh +index 2bf5f8e..10db054 100755 +--- a/btrfsmaintenance-refresh-cron.sh ++++ b/btrfsmaintenance-refresh-cron.sh +@@ -35,6 +35,18 @@ refresh_cron() { + SCRIPT="$2" + echo "Refresh script $SCRIPT for $EXPECTED" + ++ valid=false ++ for PERIOD in daily weekly monthly none uninstall; do ++ if [ "$PERIOD" = "$EXPECTED" ]; then ++ valid=true ++ fi ++ done ++ ++ if ! $valid; then ++ echo "$EXPECTED is not a valid period for cron. Not changing." ++ return ++ fi ++ + for PERIOD in daily weekly monthly; do + # NOTE: debian does not allow filenames with dots in /etc/cron.* + LINK="${SCRIPT%.*}" +-- +2.16.4 + diff --git a/0005-btrfs-defrag-plugin-remove-dependency-on-zypp-plugin.patch b/0005-btrfs-defrag-plugin-remove-dependency-on-zypp-plugin.patch new file mode 100644 index 0000000..3c2010a --- /dev/null +++ b/0005-btrfs-defrag-plugin-remove-dependency-on-zypp-plugin.patch @@ -0,0 +1,131 @@ +From a1fdf5f2502ead429fe6ea6b44a63ad35aae7242 Mon Sep 17 00:00:00 2001 +From: Jeff Mahoney +Date: Tue, 12 Mar 2019 16:22:08 -0400 +Subject: btrfs-defrag-plugin: remove dependency on zypp-plugin-python +References: FATE#326736 jsc#SLE-4130 +Notes: The script is included separately since rpm can't use git binary diffs + +Requiring zypp-plugin-python means that we need to pull in 150 MB of +python dependencies. It's present already on many systems but on +JeOS systems, it won't be by default. We still want to defrag +the RPM database on those systems. + +This commit replaces the python script with a shell script that provides +the same functionality and drops the python dependency entirely. + +Signed-off-by: Jeff Mahoney +--- + + README.md | 4 +- + btrfs-defrag-plugin.py | 81 ------------------------------------------------- + 2 files changed, 2 insertions(+), 83 deletions(-) + +diff --git a/README.md b/README.md +index d59f3d6..2202ebf 100644 +--- a/README.md ++++ b/README.md +@@ -188,7 +188,7 @@ do manual installation of files as described below. + * `sysconfig.btrfsmaintenance` configuration template is put to: + * `/etc/sysconfig/btrfsmaintenance` on SUSE and RedHat based systems or derivatives + * `/etc/default/btrfsmaintenance` on Debian and derivatives +-* `/usr/lib/zypp/plugins/commit/btrfs-defrag-plugin.py` post-update script for ++* `/usr/lib/zypp/plugins/commit/btrfs-defrag-plugin.sh` post-update script for + zypper (the package manager), applies to SUSE-based distros for now + * cron refresh scripts are installed (see bellow) + +@@ -212,7 +212,7 @@ configuration file in `/etc/sysconfig/btrfsmaintenance` by installing the + The package database files tend to be updated in a random way and get + fragmented, which particularly hurts on btrfs. For rpm-based distros this means files + in `/var/lib/rpm`. The script or plugin simpy runs a defragmentation on the affected files. +-See `btrfs-defrag-plugin.py` for more details. ++See `btrfs-defrag-plugin.sh` for more details. + + At the moment the 'zypper' package manager plugin exists. As the package + managers differ significantly, there's no single plugin/script to do that. +diff --git a/btrfs-defrag-plugin.py b/btrfs-defrag-plugin.py +deleted file mode 100644 +index dab9556..0000000 +--- a/btrfs-defrag-plugin.py ++++ /dev/null +@@ -1,81 +0,0 @@ +-#!/usr/bin/python +- +-# This plugin defragments rpm files after update. +-# +-# If the filesystem is btrfs, run defrag command in the RPM database +-# folder, set the desired extent size to 32MiB, but this may change in the +-# result depending on the fragmentation of the free space. +-# +-# Why 32MiB: +-# - the worst fragmentation has been observed on Packages +-# - this can grow up to several hundred of megabytes +-# - the file gets updated at random places +-# - although the file will be composed of many extents, it's faster to +-# merge only the extents that affect some portions of the file, instead +-# of the whole file; the difference is negligible +-# - due to the free space fragmentation over time, it's hard to find +-# contiguous space, the bigger the extent is, the worse and the extent +-# size hint is not reached anyway +- +-import sys +-if sys.version_info[0] >= 3: +- from builtins import str +- popen_kwargs = { 'encoding': 'ascii' } +-else: +- popen_kwargs = { } +-from zypp_plugin import Plugin +-import subprocess +- +-DEBUG=False +-EXTENT_SIZE=32*1024*1024 +-LOGFILE='/tmp/btrfs-defrag-plugin.log' +-PATH=subprocess.check_output(["rpm", "--eval", "%_dbpath"], **popen_kwargs).strip() +- +-def dbg(args): +- if not DEBUG: return +- f=open(LOGFILE, "a+") +- f.write(args) +- f.write("\n") +- f.close() +- +-def qx(args): +- out=subprocess.Popen(args, shell=True, stdout=subprocess.PIPE, **popen_kwargs).stdout +- outstr="".join(out.readlines()) +- out.close() +- return outstr +- +-def fstype(path): +- ret=qx('stat -f --format=%T "'+path+'"') +- return ret.rstrip() +- +-class BtrfsDefragPlugin(Plugin): +- def PLUGINBEGIN(self, headers, body): +- self.actions = [] +- self.commit_hook_supported = False +- dbg('--- Btrfs defrag plugin begin') +- self.ack() +- +- def PLUGINEND(self, headers, body): +- dbg('--- Btrfs defrag plugin end: %s %s\n' % (str(headers), str(body))) +- dbg('--- fstype(%s) = |%s|' % (PATH, fstype(PATH))) +- if fstype(PATH) != 'btrfs': +- self.ack() +- return +- if DEBUG: +- dbg('--- Fragmentation before') +- dbg(qx('filefrag %s/*' % (PATH))) +- # defrag options: +- # - verbose +- # - recursive +- # - flush each file before going to the next one +- # - set the extent target hint +- ret = qx('btrfs filesystem defragment -v -f -r -t %s "%s"' % \ +- (str(EXTENT_SIZE), PATH)) +- if DEBUG: +- dbg(ret) +- dbg('--- Fragmentation after') +- dbg(qx('filefrag %s/*' % (PATH))) +- self.ack() +- +-plugin = BtrfsDefragPlugin() +-plugin.main() diff --git a/btrfs-defrag-plugin.sh b/btrfs-defrag-plugin.sh new file mode 100644 index 0000000..0c7f20e Binary files /dev/null and b/btrfs-defrag-plugin.sh differ diff --git a/btrfsmaintenance.changes b/btrfsmaintenance.changes index 481c2e6..2d99050 100644 --- a/btrfsmaintenance.changes +++ b/btrfsmaintenance.changes @@ -1,3 +1,24 @@ +------------------------------------------------------------------- +Tue Mar 12 20:37:45 UTC 2019 - jeffm@suse.com + +- Document and adjust balance thresholds and periods (FATE#325529 jsc#SLE-3188). + * README: document impact of balance + quota groups + * sysconfig: reduce default balance usage thresholds + * sysconfig: document using systemd.time(7) for frequency of operations + * btrfsmaintenance-refresh-cron.sh: validate periods for cron-based systems +- Remove python dependency (FATE#326736 jsc#SLE-4130). + * btrfs-defrag-plugin: remove dependency on zypp-plugin-python +- Added patches: + * 0001-README-document-impact-of-balance-quota-groups.patch + * 0002-sysconfig-reduce-default-balance-usage-thresholds-fa.patch + * 0003-sysconfig-document-using-systemd.time-7-for-frequenc.patch + * 0004-btrfsmaintenance-refresh-cron.sh-validate-periods-fo.patch + * 0005-btrfs-defrag-plugin-remove-dependency-on-zypp-plugin.patch +- Added source: + * btrfs-defrag-plugin.sh +- Removed patch: + * python3-support-bsc1070322.diff + ------------------------------------------------------------------- Tue Sep 25 00:00:00 CEST 2018 - dsterba@suse.cz diff --git a/btrfsmaintenance.spec b/btrfsmaintenance.spec index 003cf49..4d69cd2 100644 --- a/btrfsmaintenance.spec +++ b/btrfsmaintenance.spec @@ -1,7 +1,7 @@ # # spec file for package btrfsmaintenance # -# Copyright (c) 2018 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2019 SUSE LINUX GmbH, Nuernberg, Germany. # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -29,10 +29,14 @@ License: GPL-2.0-only Group: System/Base Url: https://github.com/kdave/btrfsmaintenance Source0: %{name}-%{version}.tar.bz2 -Patch0: python3-support-bsc1070322.diff +Source1: btrfs-defrag-plugin.sh +Patch1: 0001-README-document-impact-of-balance-quota-groups.patch +Patch2: 0002-sysconfig-reduce-default-balance-usage-thresholds-fa.patch +Patch3: 0003-sysconfig-document-using-systemd.time-7-for-frequenc.patch +Patch4: 0004-btrfsmaintenance-refresh-cron.sh-validate-periods-fo.patch +Patch5: 0005-btrfs-defrag-plugin-remove-dependency-on-zypp-plugin.patch BuildRequires: systemd Requires: btrfsprogs -Requires: python3-zypp-plugin Requires: libzypp(plugin:commit) Supplements: btrfsprogs BuildArch: noarch @@ -45,7 +49,12 @@ for snapper). %prep %setup -q -%patch0 -p1 +%patch1 -p1 +%patch2 -p1 +%patch3 -p1 +%patch4 -p1 +%patch5 -p1 +cp %{SOURCE1} . %build @@ -76,7 +85,7 @@ ln -s %{_sbindir}/service %{buildroot}%{_sbindir}/rcbtrfsmaintenance-refresh # zypp plugin install -m 755 -d %{buildroot}%{_libexecdir}/zypp/plugins/commit -install -m 755 -D btrfs-defrag-plugin.py %{buildroot}%{_libexecdir}/zypp/plugins/commit +install -m 755 -D btrfs-defrag-plugin.sh %{buildroot}%{_libexecdir}/zypp/plugins/commit # config install -m 755 -d %{buildroot}%{_fillupdir} @@ -107,7 +116,7 @@ install -m 644 -D sysconfig.btrfsmaintenance %{buildroot}%{_fillupdir} %dir %{_libexecdir}/zypp/ %dir %{_libexecdir}/zypp/plugins %dir %{_libexecdir}/zypp/plugins/commit -%{_libexecdir}/zypp/plugins/commit/btrfs-defrag-plugin.py +%{_libexecdir}/zypp/plugins/commit/btrfs-defrag-plugin.sh %{_unitdir}/btrfsmaintenance-refresh.path %{_unitdir}/btrfsmaintenance-refresh.service %{_unitdir}/btrfs-balance.service diff --git a/python3-support-bsc1070322.diff b/python3-support-bsc1070322.diff deleted file mode 100644 index 0d27a1c..0000000 --- a/python3-support-bsc1070322.diff +++ /dev/null @@ -1,12 +0,0 @@ -As per explanation in https://bugzilla.suse.com/show_bug.cgi?id=1070322#c10 -python3 must be called. -Index: btrfsmaintenance-0.4.1/btrfs-defrag-plugin.py -=================================================================== ---- btrfsmaintenance-0.4.1.orig/btrfs-defrag-plugin.py -+++ btrfsmaintenance-0.4.1/btrfs-defrag-plugin.py -@@ -1,4 +1,4 @@ --#!/usr/bin/python -+#!/usr/bin/python3 - - # This plugin defragments rpm files after update. - #