Accepting request 234945 from home:garloff:branches:Base:System

Update to 1.43, with a number of improvements (e.g. ARMv8 optimized zero-block detection) and most importantly the dd_rescue-lzo subpackage that provides a dd_rescue plugin with lzo de/compression support.

OBS-URL: https://build.opensuse.org/request/show/234945
OBS-URL: https://build.opensuse.org/package/show/Base:System/dd_rescue?expand=0&rev=22

dd_rescue-1.42.tar.gz

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:740fc84b167bea4cecae3b22e0e87f11ef0e563710c7cd8c5bb79b1eebfcbd2f
-size 69695

dd_rescue-1.43-1.43.1.diff

@@ -0,0 +1,150 @@
+Index: Makefile
+===================================================================
+RCS file: /share/MD0_DATA/cvsroot/dd_rescue/Makefile,v
+retrieving revision 1.153
+retrieving revision 1.154
+diff -u -p -r1.153 -r1.154
+--- Makefile	20 May 2014 07:28:01 -0000	1.153
++++ Makefile	21 May 2014 08:27:22 -0000	1.154
+@@ -1,6 +1,6 @@
+ # Makefile for dd_rescue
+ # (c) garloff@suse.de, 99/10/09, GNU GPL
+-# $Id: Makefile,v 1.153 2014/05/20 07:28:01 garloff Exp $
++# $Id: Makefile,v 1.154 2014/05/21 08:27:22 garloff Exp $
+ 
+ VERSION = 1.43
+ 
+@@ -15,7 +15,8 @@ INSTALLFLAGS = -s
+ prefix = $(DESTDIR)/usr
+ INSTALLDIR = $(prefix)/bin
+ #INSTALLDIR = $(DESTDIR)/bin
+-INSTALLLIBDIR = $(prefix)/$(LIB)
++#INSTALLLIBDIR = $(prefix)/$(LIB)
++INSTALLLIBDIR = $(DESTDIR)/$(LIBDIR)
+ MANDIR = $(prefix)/share/man
+ #MYDIR = dd_rescue-$(VERSION)
+ MYDIR = dd_rescue
+@@ -279,11 +280,11 @@ check_lzo: $(TARGETS)
+ 	$(LZOP) -fd dd_rescue.ddr.lzo
+ 	cmp dd_rescue dd_rescue.ddr
+ 	@rm -f dd_rescue.ddr dd_rescue.ddr.lzo
+-	./dd_rescue -b1M -L ./libddr_MD5.so=output,./libddr_lzo.so=compress,./libddr_MD5.so=output dd_rescue dd_rescue.ddr.lzo > dd_rescue.ddr.MD5SUM
++	./dd_rescue -b256k -L ./libddr_MD5.so=output,./libddr_lzo.so=compress,./libddr_MD5.so=output dd_rescue dd_rescue.ddr.lzo > dd_rescue.ddr.MD5SUM
+ 	md5sum -c dd_rescue.ddr.MD5SUM
+ 	md5sum dd_rescue dd_rescue.ddr.lzo
+ 	$(LZOP) -Nvl dd_rescue.ddr.lzo
+-	./dd_rescue -b1M -TL ./libddr_MD5.so=output,./libddr_lzo.so=compress,./libddr_MD5.so,./libddr_lzo.so=decompress,./libddr_MD5.so=outfd=1 dd_rescue dd_rescue.ddr > dd_rescue.ddr.MD5
++	./dd_rescue -b256k -TL ./libddr_MD5.so=output,./libddr_lzo.so=compress,./libddr_MD5.so,./libddr_lzo.so=decompress,./libddr_MD5.so=outfd=1 dd_rescue dd_rescue.ddr > dd_rescue.ddr.MD5
+ 	cmp dd_rescue dd_rescue.ddr
+ 	md5sum -c dd_rescue.ddr.MD5
+ 	./dd_rescue -b16k -TL ./libddr_MD5.so=output,./libddr_lzo.so=compress,./libddr_MD5.so,./libddr_lzo.so=decompress,./libddr_MD5.so=outfd=1 dd_rescue dd_rescue.ddr > dd_rescue.ddr.MD5
+@@ -292,7 +293,7 @@ check_lzo: $(TARGETS)
+ 	@cp -p dd_rescue.ddr.lzo dd_rescue.lzo
+ 	@rm -f dd_rescue.ddr dd_rescue.ddr.lzo dd_rescue.ddr.MD5
+ 	$(LZOP) -f dd_rescue
+-	./dd_rescue -b1M -TL ./libddr_lzo.so dd_rescue.lzo dd_rescue.cmp
++	./dd_rescue -b256k -TL ./libddr_lzo.so dd_rescue.lzo dd_rescue.cmp
+ 	cmp dd_rescue dd_rescue.cmp
+ 	@rm -f dd_rescue.cmp dd_rescue.lzo
+ 	./dd_rescue -b16k -L ./libddr_MD5.so=output,./libddr_lzo.so,./libddr_MD5.so=output dd_rescue dd_rescue.lzo > MD5.1
+Index: ddr_lzo.1
+===================================================================
+RCS file: /share/MD0_DATA/cvsroot/dd_rescue/ddr_lzo.1,v
+retrieving revision 1.8
+retrieving revision 1.9
+diff -u -p -r1.8 -r1.9
+--- ddr_lzo.1	19 May 2014 19:59:25 -0000	1.8
++++ ddr_lzo.1	21 May 2014 08:27:22 -0000	1.9
+@@ -1,4 +1,4 @@
+-.\" $Id: ddr_lzo.1,v 1.8 2014/05/19 19:59:25 garloff Exp $
++.\" $Id: ddr_lzo.1,v 1.9 2014/05/21 08:27:22 garloff Exp $
+ .
+ .TH ddr_lzo 1 "2014-05-12" "Kurt Garloff" "LZO de/compression plugin for dd_rescue"
+ .
+@@ -176,10 +176,11 @@ resulting lzop file to ddr_lzo's own ver
+ incompatibility with lzop (as of 1.03).
+ .br
+ lzop by default uses block sizes of 256kiB (on Unix systems), but
+-supports de/compression with different block sizes as well. The
+-maximum possible is 64MiB; for reasons of error resilience, it is
+-however NOT recommended to use blocks larger than 16MiB, see
+-below.
++supports de/compression with smaller block sizes as well. It needs
++to be recompiled to support block sizes up to a possible maximum
++of 64MiB. Thus staying below or at 256kiB is recommended; even
++when lzop compatibility is no concern, blocks larger than 16MiB
++are not recommended, see below.
+ .
+ .SS Blocksize considerations
+ When decompressing, the (soft) block size chosen in dd_rescue must be 
+@@ -196,8 +197,10 @@ The default from dd_rescue (128kiB) is a
+ recommended to increase the block size too much -- when an lzo file gets
+ corrupted, at least one block will be lost; larger blocks result in larger
+ damage. Also, blocks larger than 16MiB will not work well with the error
+-tolerance features of ddr_lzo. Also note that blocks larger than 64MiB
+-prevent decompression with lzop.
++tolerance features of ddr_lzo. Also note that blocks larger than 256kiB
++need recompilation of lzop if you want to be able to use lzop to
++process the .lzo files; blocks larger than 64MiB prevent decompression
++even with a recompiled lzop.
+ .
+ .SH BUGS/LIMITATIONS
+ .SS Maturity
+@@ -205,7 +208,7 @@ The plugin is new as of dd_rescue 1.43. 
+ saved with ddr_lzo as the only backup for valuable data. Also
+ expect some changes to ddr_lzo in the not too distant future. 
+ (This should not break the file format, as we're following lzop ....)
+-.br 
++.br
+ Compressed data is more sensitive to data corruption than plain data.
+ Note that the checksums (adler32 or crc32) in the lzop file format
+ do NOT allow to correct for errors; they just allow a somewhat reliable
+Index: libddr_lzo.c
+===================================================================
+RCS file: /share/MD0_DATA/cvsroot/dd_rescue/libddr_lzo.c,v
+retrieving revision 1.107
+retrieving revision 1.110
+diff -u -p -r1.107 -r1.110
+--- libddr_lzo.c	20 May 2014 09:12:23 -0000	1.107
++++ libddr_lzo.c	21 May 2014 09:22:01 -0000	1.110
+@@ -6,6 +6,10 @@
+  * License: GNU GPLv2 or v3
+  */
+ 
++#ifndef _GNU_SOURCE
++# define _GNU_SOURCE 1
++#endif
++
+ #define _LARGEFILE64_SOURCE 1
+ #define _FILE_OFFSET_BITS 64
+ 
+@@ -41,9 +45,9 @@
+ #endif
+ 
+ #ifdef HAVE_BASENAME
+-const char* basename(const char*);
++//const char* basename(const char*);
+ #else
+-static char* basename(const char *nm)
++static const char* basename(const char *nm)
+ {
+ 	const char* ptr = strrchr(nm, '/');	/* Not on DOS */
+ 	if (ptr)
+@@ -531,9 +535,14 @@ int lzo_open(const opt_t *opt, int ilnch
+ 	state->dbuf = (unsigned char*)slackalloc(state->dbuflen, state);
+ 	if (state->do_bench) 
+ 		state->cpu = 0;
+-	if (opt->softbs > MAXBLOCKSZ)
+-		FPLOG(WARN, "Blocks larger than %iMiB not recommended %iMiB specified)\n",
+-			MAXBLOCKSZ>>20, opt->softbs);
++	if (state->mode == COMPRESS) {
++		if (opt->softbs > MAXBLOCKSZ)
++			FPLOG(WARN, "Blocks larger than %iMiB not recommended (%iMiB specified)\n",
++				MAXBLOCKSZ>>20, opt->softbs>>20);
++		else if (opt->softbs > 256*1024)
++			FPLOG(WARN, "Blocks larger than 256kiB need recompilation of lzop (%ikiB specified)\n",
++				opt->softbs>>10);
++	}
+ 	state->next_ipos = opt->init_ipos;
+ 	return 0;
+ 	/* This breaks MD5 in chain before us

dd_rescue-1.43.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:661b5e9aa0dcbf1213ba33a727df1ff1da834775dbef97c11b789bc938ff8405
+size 107135

dd_rescue-fix_insn_probe.diff

@@ -1,51 +0,0 @@
-From: Kurt Garloff <kurt@garloff.de>
-Subject: Need to restore sigprocmask when not returning from signal handler
-
-I stumbled over the fine details of signal handling:
-A second SIGILL would not have been delivered to the handler as it was
-blocked when invoking the handler for the first time. As we can't return
-from the handler, we need to restore the process' signal mask ourselves
-and reenable delivery.
-
-Signed-off-by: Kurt Garloff <kurt@garloff.de>
-
-Index: find_nonzero.c
-===================================================================
-RCS file: /home/cvsroot/dd_rescue/find_nonzero.c,v
-retrieving revision 1.44
-retrieving revision 1.46
-diff -u -p -r1.44 -r1.46
---- find_nonzero.c	21 Feb 2014 12:51:43 -0000	1.44
-+++ find_nonzero.c	6 Mar 2014 12:14:18 -0000	1.46
-@@ -24,6 +24,14 @@ static jmp_buf sigill_jmp;
- static void ill_handler(int sig)
- {
- 	have_feature = 0;
-+	/* As we can't return from handler (as it would result in 
-+	 * reexecuting the illegal instruction again - we jump back
-+	 * using longjmp) -- we have to restore signal delivery, so the
-+	 * program context is back to normal. Otherwise a second
-+	 * probe_procedure would not handle SIGILL. */
-+	sigset_t sigmask;
-+	sigemptyset(&sigmask); sigaddset(&sigmask, sig);
-+	sigprocmask(SIG_UNBLOCK, &sigmask, NULL);
- 	longjmp(sigill_jmp, 1);
- }
- 
-Index: archdep.h
-===================================================================
-RCS file: /home/cvsroot/dd_rescue/archdep.h,v
-retrieving revision 1.6
-retrieving revision 1.7
-diff -u -p -r1.6 -r1.7
---- archdep.h	25 Feb 2014 16:06:35 -0000	1.6
-+++ archdep.h	6 Mar 2014 12:20:46 -0000	1.7
-@@ -28,7 +28,7 @@ void probe_sse2();
- #define have_avx2 0
- #define have_sse42 0
- #define ARCH_DETECT do {} while (0)
--#define ARCH_DECLS
-+#define ARCH_DECLS ARCH_DECL_386
- #elif defined(NO_AVX2) /* compiler does not support -mavx2 */
- #define have_avx2 0
- extern char have_sse42;

dd_rescue.changes

@@ -1,3 +1,37 @@
+-------------------------------------------------------------------
+Wed May 21 10:44:18 CEST 2014 - kurt@garloff.de
+
+- dd_rescue-1.43-1.43.1.diff: Document 256kiB limit of lzop in
+  man page, output warning in ddr_lzo and respect limit in test
+  suite.
+
+-------------------------------------------------------------------
+Tue May 20 09:29:40 CEST 2014 - kurt@garloff.de
+
+- Update to 1.43-final: Make make check succeed without lzop.
+
+-------------------------------------------------------------------
+Mon May 19 21:03:04 CEST 2014 - kurt@garloff.de
+
+- Update to 1.43-pre:
+  * Fix bug in extend, which previously would disallow to append
+    more data than existed before.
+  * Some minor improvements on messages and manpage.
+  * ARMv8 (AArch64) optimized find_nonzero for fast zero-block
+    detection.
+  * Significant rework of plugin interface, leading also to some
+    refactoring of option parsing and passing. 
+  * Refactoring of MD5 plugin, improving tolerance against holes
+    e.g. at the end of a file. Better test coverage. Consistent
+    message prefix. output option to produce md5sum check files.
+  * Add libddr_lzo plugin, allowing transparent de/compression
+    of the data. Supports sparse files, appending, ... and a variety
+    of algorithms and has a number of ways to handle somewhat
+    damaged files. With the lzo1x_ algorithms, it's compatible with
+    lzop. Comes with a good number of tests and an own fuzzer to
+    test robustness of the decompressor. Shipped with an own man 
+    page and is packaged into the dd_rescue-lzo sub package.
+
 -------------------------------------------------------------------
 Sat Mar  8 18:58:56 CET 2014 - kurt@garloff.de
 

dd_rescue.spec

@@ -17,21 +17,23 @@
 
 
 Name:           dd_rescue
-Version:        1.42
+Version:        1.43
 Release:        0
 Summary:        Data Copying in the Presence of I/O Errors
 License:        GPL-2.0 or GPL-3.0
 Group:          System/Base
 Url:            http://www.garloff.de/kurt/linux/ddrescue/
 Source0:        http://garloff.de/kurt/linux/ddrescue/%{name}-%{version}.tar.gz
-Patch0:         dd_rescue-fix_insn_probe.diff
+Patch0:         http://garloff.de/kurt/linux/ddrescue/%{name}-%{version}-%{version}.1.diff
 BuildRequires:  autoconf
 BuildRequires:  libattr-devel
+BuildRequires:  lzo-devel
 %if 0%{?suse_version} >= 1200
 BuildRequires:  libfallocate-devel
+BuildRequires:  lzop
 %endif
 Requires:       bc
-Recommends:     dd_rhelp libfallocate0
+Recommends:     dd_rhelp libfallocate0 dd_rescue-lzo
 # ddrescue was last used in openSUSE 11.4 (version 1.14_0.0.6)
 Provides:       ddrescue = %{version}
 Obsoletes:      ddrescue < %{version}
@@ -45,11 +47,43 @@ cat, and dd will "abort" on every I/O error, dd_rescue does not.
 dd_rescue has many other goodies; optimization by using large blocks
 as long as no errors are in sight and falling back to small ones; reverse
 direction copy; splice in-kernel zerocopy; O_DIRECT support; preallocation
-with fallocate(); random number writing etc.
+with fallocate().
+
+dd_rescue also provides data protection features by overwriting files
+or disks with fast random numbers, optionally multiple times.
+
+dd_rescue supports plugins; currently an MD5 and an lzo plugin exist.
+
+%package lzo
+Summary:        LZO plugin for dd_rescue
+Group:          System/Base
+Requires:       dd_rescue = %{version}
+
+%description lzo
+This plugin allows you do de/compress files during recovery copying
+with dd_rescue using the lzo family of algorithms. lzo algorithms
+are very fast to decompress and most algorithms are very fast to
+compress as well -- at the expense of somewhat worse compression than
+zlib's deflate.
+
+The plugin does offer a variety of options to handle corrupted .lzo
+files with some grace; it does skip over bad blocks (if the block
+headers are still intact) by default, but does offer an option (nodiscard)
+to allow to attempt decompression on faulty input, hoping to produce 
+some usable bytes. It can also search for valid block headers after
+synchronization has been lost due to a corrupt one.
+
+The plugin also handles sparse files (files with holes) and supports
+appending to .lzo files, so it fits neatly into dd_rescue.
+
+Some fuzz testing has been applied to the plugin's decompression routines,
+though more will have to be done to feel confident about feeding untrusted
+data to the decompressor; the plugin is still young and might expose bugs.
 
 %prep
 %setup -q -n dd_rescue
 %patch0 -p0
+./autogen.sh
 
 # Remove build time references so build-compare can do its work
 FAKE_BUILDTIME=$(LC_ALL=C date -u -r %{_sourcedir}/%{name}.changes '+%%H:%%M')
@@ -59,10 +93,10 @@ sed -i "s/__DATE__/\"$FAKE_BUILDDATE\"/g" dd_rescue.c
 
 %build
 flags="%{optflags}"
-make RPM_OPT_FLAGS="$flags" LIBDIR=%{_libdir} %{?_smp_mflags}
+make RPM_OPT_FLAGS="$flags" LIBDIR=%{_libdir} LIB=%{_lib} %{?_smp_mflags}
 
 %install
-make install DESTDIR=%{buildroot} INSTALLDIR=%{buildroot}/%{_bindir} LIBDIR=%{_libdir} \
+make install DESTDIR=%{buildroot} INSTALLDIR=%{buildroot}/%{_bindir} LIB=%{_lib} LIBDIR=%{_libdir} \
     INSTASROOT= INSTALLFLAGS=
 
 #UsrMerge
@@ -83,4 +117,9 @@ make check
 %{_libdir}/libddr_MD5.so
 %doc %{_mandir}/man1/dd_rescue.1%{ext_man}
 
+%files lzo
+%defattr(-,root,root,-)
+%{_libdir}/libddr_lzo.so
+%doc %{_mandir}/man1/ddr_lzo.1%{ext_man}
+
 %changelog