draigBrady.com> +Subject: Re: coreutils-8.10 released [stable] +Newsgroups: gmane.comp.gnu.coreutils.general +Date: 2011-02-07 12:39:27 GMT (2 days, 22 hours and 55 minutes ago) + +On 05/02/11 13:59, Jim Meyering wrote: +> Pádraig Brady wrote: +>> Yep, just did that, and got a couple of failures for sparse-fiemap +>> on and ext3 and loopback ext4 file systems. +>> On a very quick glance, I think cp is OK and that the filefrag +>> matching is a bit brittle. +>> Attached are filefrag outputs. +> +> I saw similar differences, but I think they were due to the fact that +> the kernel had not yet forced cp's metadata update to disk when filefrag +> does its FIEMAP ioctl +> +> Uncommenting the "sync" just after the "cp" in the sparse-fiemap test +> solved the problem (for me it arose only on rawhide's btrfs) but made +> the test take a lot more time, as mentioned in the comment. + +It seems that the sync is needed for ext4 loopback also, on my system. +However for ext3, the extents between src and dst still don't match up. +My systems uses a 4K block size and is 2.6.35.10-72.fc14.i686 +For now, I've disabled (indirectly) using ext3 for fiemap-perf and +sparse-fiemap in the attached. + +> Instead of that sync, using filefrag's -s option may +> have the same result without the unwanted overhead. + +That doesn't work actually which baffles me. +I got the e2fsprogs source rpm to verify that +FIEMAP_FLAG_SYNC was set, so I guess this is a kernel issue +(on ext4 loop back at least). +Though I was able to get a a working sync restricted to the file +by using dd in the attached patch. +Even with that, the test only takes about 10s on my old laptop, so +I didn't bother tagging the test as EXPENSIVE. + +cheers, +Pádraig. + +From 1da62d67ce4d4c78b98bc947c9367a10f1bdba9d Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?P=C3=A1draig=20Brady?=
draigBrady.com> +Date: Fri, 4 Feb 2011 22:05:20 +0000 +Subject: [PATCH] test: improve the fiemap_capable_ check + +* tests/cp/fiemap-2: Enable the fiemap check for files, which +will enable the test on ext3. +* tests/cp/fiemap-perf: Comment why we're not enabling for ext3. +* tests/cp/sparse-fiemap: Ditto. +* tests/fiemap-capable: A new python script to determine +if a specified path supports fiemap. +* tests/init.cfg (fiemap_capable_): Use the new python script. +* tests/Makefile.am (EXTRA_DIST): Include the new python script. +--- + tests/Makefile.am | 1 + + tests/cp/fiemap-2 | 3 ++- + tests/cp/fiemap-perf | 2 ++ + tests/cp/sparse-fiemap | 12 ++++++++---- + tests/fiemap-capable | 16 ++++++++++++++++ + tests/init.cfg | 9 ++++----- + 6 files changed, 33 insertions(+), 10 deletions(-) + create mode 100644 tests/fiemap-capable + +diff --git a/tests/Makefile.am b/tests/Makefile.am +index 751b327..8aa56cd 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -11,6 +11,7 @@ EXTRA_DIST = \ + check.mk \ + envvar-check \ + filefrag-extent-compare \ ++ fiemap-capable \ + init.cfg \ + init.sh \ + lang-default \ +diff --git a/tests/cp/fiemap-2 b/tests/cp/fiemap-2 +index a17076c..691ead2 100755 +--- a/tests/cp/fiemap-2 ++++ b/tests/cp/fiemap-2 +@@ -20,7 +20,8 @@ + print_ver_ cp + + # Require a fiemap-enabled FS. +-fiemap_capable_ . \ ++touch fiemap_chk # check a file rather than current dir for best coverage ++fiemap_capable_ fiemap_chk \ + || skip_ "this file system lacks FIEMAP support" + + # Exercise the code that handles a file ending in a hole. +diff --git a/tests/cp/fiemap-perf b/tests/cp/fiemap-perf +index 7369a7d..dbb2a81 100755 +--- a/tests/cp/fiemap-perf ++++ b/tests/cp/fiemap-perf +@@ -20,6 +20,8 @@ + print_ver_ cp + + # Require a fiemap-enabled FS. ++# Note we don't check a file here as that could enable ++# the test on ext3 where emulated extent scanning can be slow. + fiemap_capable_ . \ + || skip_ "this file system lacks FIEMAP support" + +diff --git a/tests/cp/sparse-fiemap b/tests/cp/sparse-fiemap +index f224b5b..fc27869 100755 +--- a/tests/cp/sparse-fiemap ++++ b/tests/cp/sparse-fiemap +@@ -19,6 +19,8 @@ + . "${srcdir=.}/init.sh"; path_prepend_ ../src + print_ver_ cp + ++# Note we don't check a file here as that could enable ++# the test on ext3 where this test is seen to fail. + if fiemap_capable_ . ; then + : # Current dir is on a partition with working extents. Good! + else +@@ -66,11 +68,13 @@ for i in $(seq 1 2 21); do + $PERL -e 'BEGIN { $n = '$i' * 1024; *F = *STDOUT }' \ + -e 'for (1..'$j') { sysseek (*F, $n, 1)' \ + -e '&& syswrite (*F, chr($_)x$n) or die "$!"}' > j1 || fail=1 +- # sync ++ ++ # Note the explicit fdatasync is used here as ++ # it was seen that `filefrag -s` (FIEMAP_FLAG_SYNC) was ++ # ineffective on ext4 loopback on Linux 2.6.35.10-72.fc14.i686 ++ dd if=/dev/null of=j1 conv=notrunc,fdatasync + cp --sparse=always j1 j2 || fail=1 +- # sync +- # Technically we may need the 'sync' uses above, but +- # uncommenting them makes this test take much longer. ++ dd if=/dev/null of=j2 conv=notrunc,fdatasync + + cmp j1 j2 || fail=1 + filefrag -v j1 | grep extent \ +diff --git a/tests/fiemap-capable b/tests/fiemap-capable +new file mode 100644 +index 0000000..05c6926 +--- /dev/null ++++ b/tests/fiemap-capable +@@ -0,0 +1,16 @@ ++import struct, fcntl, sys, os ++ ++def sizeof(t): return struct.calcsize(t) ++IOCPARM_MASK = 0x7f ++IOC_OUT = 0x40000000 ++IOC_IN = 0x80000000 ++IOC_INOUT = (IOC_IN|IOC_OUT) ++def _IOWR(x,y,t): return (IOC_INOUT|((sizeof(t)&IOCPARM_MASK)<<16)|((x)<<8)|y) ++ ++try: ++ fd = os.open (len (sys.argv) == 2 and sys.argv[1] or '.', os.O_RDONLY) ++ struct_fiemap = '=qqllll' ++ FS_IOC_FIEMAP = _IOWR (ord ('f'), 11, struct_fiemap) ++ fcntl.ioctl (fd, FS_IOC_FIEMAP, struct.pack(struct_fiemap, 0,~0,0,0,0,0)) ++except: ++ sys.exit (1) +diff --git a/tests/init.cfg b/tests/init.cfg +index b2d1bab..f49c5cf 100644 +--- a/tests/init.cfg ++++ b/tests/init.cfg +@@ -295,13 +295,12 @@ require_proc_pid_status_() + kill $pid + } + +-# Return nonzero if the specified directory is on a file system for +-# which FIEMAP support exists, and the file system type is new enough +-# (unlike ext2 and ext3) that it is hard to find an instance *without* +-# FIEMAP support. ++# Return nonzero if the specified path is on a file system for ++# which FIEMAP support exists. Note some file systems (like ext3) ++# only support FIEMAP for files, not directories. + fiemap_capable_() + { +- df -T -t btrfs -t xfs -t ext4 -t ocfs2 -t gfs2 "$@" ++ python $abs_srcdir/fiemap-capable "$@" + } + + # Does the current (working-dir) file system support sparse files? +-- +1.7.3.4 + diff --git a/coreutils.changes b/coreutils.changes index 217a818..ef2de1d 100644 --- a/coreutils.changes +++ b/coreutils.changes @@ -1,3 +1,51 @@ +------------------------------------------------------------------- +Thu Feb 10 14:35:49 CET 2011 - pth@suse.de + +- Update to 8.10: + * Bug fixes + + - du would abort with a failed assertion when two conditions are + met: part of the hierarchy being traversed is moved to a higher + level in the directory tree, and there is at least one more + command line directory argument following the one containing + the moved sub-tree. [bug introduced in coreutils-5.1.0] + + - join --header now skips the ordering check for the first line + even if the other file is empty. [bug introduced in + coreutils-8.5] + + - rm -f no longer fails for EINVAL or EILSEQ on file systems that + reject file names invalid for that file system. + + - uniq -f NUM no longer tries to process fields after end of + line. [bug introduced in coreutils-7.0] + + * New features + + - cp now copies sparse files efficiently on file systems with + FIEMAP support (ext4, btrfs, xfs, ocfs2). Before, it had to + read 2^20 bytes when copying a 1MiB sparse file. Now, it + copies bytes only for the non-sparse sections of a file. + Similarly, to induce a hole in the output file, it had to + detect a long sequence of zero bytes. Now, it knows precisely + where each hole in an input file is, and can reproduce them + efficiently in the output file. mv also benefits when it + resorts to copying, e.g., between file systems. + + - join now supports -o 'auto' which will automatically infer the + output format from the first line in each file, to ensure the + same number of fields are output for each line. + + * Changes in behavior + + - join no longer reports disorder when one of the files is empty. + This allows one to use join as a field extractor like: + join -a1 -o 1.3,1.1 - /dev/null + +- Add upstream patch that fixes a segfault in cut. +- Add upstream patch to fix sparse fiemap tests. +- Fix i18n patch for join. + ------------------------------------------------------------------- Fri Jan 14 14:13:28 CET 2011 - uli@suse.de diff --git a/coreutils.spec b/coreutils.spec index 8bd0d18..eea7759 100644 --- a/coreutils.spec +++ b/coreutils.spec @@ -23,8 +23,8 @@ BuildRequires: help2man libacl-devel libcap-devel libselinux-devel pam-devel xz Url: http://www.gnu.org/software/coreutils/ License: GFDLv1.2 ; GPLv2+ ; GPLv3+ Group: System/Base -Version: 8.9 -Release: 2 +Version: 8.10 +Release: 0 Provides: fileutils = %{version}, sh-utils = %{version}, stat = %version}, textutils = %{version}, mktemp = %{version} Obsoletes: fileutils < %{version}, sh-utils < %{version}, stat < %version}, textutils < %{version}, mktemp < %{version} Obsoletes: libselinux <= 1.23.11-3 libselinux-32bit = 9 libselinux-64bit = 9 libselinux-x86 = 9 @@ -36,7 +36,6 @@ Source: coreutils-%{version}.tar.xz Source1: su.pamd Source2: su.default Source3: baselibs.conf -Source4: coreutils-8.9.de.po Patch0: coreutils-misc.patch Patch1: coreutils-no_hostname_and_hostid.patch Patch2: coreutils-gl_printf_safe.patch @@ -56,6 +55,10 @@ Patch30: coreutils-8.6-compile-su-with-fpie.diff Patch31: coreutils-getaddrinfo.patch Patch32: coreutils-ptr_int_casts.patch Patch33: coreutils-8.9-singlethreaded-sort.patch +#Next two patches added after 8.10 release so should be obsolete with next release +Patch34: coreutils-compute_range_endpoint.patch +Patch35: coreutils-fix_sparse_fiemap_test.patch + BuildRoot: %{_tmppath}/%{name}-%{version}-build # this will create a cycle, broken up randomly - coreutils is just too core to have other # prerequires @@ -96,7 +99,8 @@ uname unexpand uniq unlink uptime users vdir wc who whoami yes %patch31 %patch32 %patch33 -cp %{S:4} po/de.po +%patch34 -p1 +%patch35 -p1 %build AUTOPOINT=true autoreconf -fi @@ -111,28 +115,28 @@ make %{?_smp_mflags} V=1 %install %makeinstall -test -f $RPM_BUILD_ROOT%{_bindir}/su || \ - install src/su $RPM_BUILD_ROOT%{_bindir}/su -install -d $RPM_BUILD_ROOT/bin +test -f %{buildroot}%{_bindir}/su || \ + install src/su %{buildroot}%{_bindir}/su +install -d %{buildroot}/bin for i in arch basename cat chgrp chmod chown cp date dd df echo false kill ln ls mkdir mknod mktemp mv pwd rm rmdir sleep sort stat stty su sync touch true uname readlink md5sum do - mv $RPM_BUILD_ROOT%{_bindir}/$i $RPM_BUILD_ROOT/bin/$i + mv %{buildroot}%{_bindir}/$i %{buildroot}/bin/$i test $i = su && echo -n '%%attr(4755,root,root) ' echo /bin/$i done > bin.files -ln -sf ../../bin/{basename,sort,stat,touch,readlink,md5sum} $RPM_BUILD_ROOT%{_bindir} -install -d -m 755 $RPM_BUILD_ROOT/etc/pam.d -install -m 644 $RPM_SOURCE_DIR/su.pamd $RPM_BUILD_ROOT/etc/pam.d/su -install -m 644 $RPM_SOURCE_DIR/su.pamd $RPM_BUILD_ROOT/etc/pam.d/su-l -install -d -m 755 $RPM_BUILD_ROOT/etc/default -install -m 644 $RPM_SOURCE_DIR/su.default $RPM_BUILD_ROOT/etc/default/su +ln -sf ../../bin/{basename,sort,stat,touch,readlink,md5sum} %{buildroot}%{_bindir} +install -d -m 755 %{buildroot}/etc/pam.d +install -m 644 %{S:1} %{buildroot}/etc/pam.d/su +install -m 644 %{S:1} %{buildroot}/etc/pam.d/su-l +install -d -m 755 %{buildroot}/etc/default +install -m 644 %{S:2} %{buildroot}/etc/default/su echo '.so man1/test.1' > %{buildroot}/%{_mandir}/man1/\[.1 %find_lang %name %post %install_info --info-dir=%{_infodir} %{_infodir}/coreutils.info.gz # may fail if permissions is not there, but there is no way around that -%run_permissions +%set_permissions %{_bindir}/su %postun %install_info_delete --info-dir=%{_infodir} %{_infodir}/coreutils.info.gz @@ -141,14 +145,14 @@ echo '.so man1/test.1' > %{buildroot}/%{_mandir}/man1/\[.1 %verify_permissions -e /bin/su %clean -rm -rf $RPM_BUILD_ROOT +rm -rf %{buildroot} %files -f bin.files %defattr(-,root,root) %doc README NEWS -%config /etc/pam.d/su -%config /etc/pam.d/su-l -%config(noreplace) /etc/default/su +%config %{_sysconfdir}/pam.d/su +%config %{_sysconfdir}/pam.d/su-l +%config(noreplace) %{_sysconfdir}/default/su %{_bindir}/* %{_libdir}/%{name} %doc %{_infodir}/coreutils.info*.gz