Accepting request 928134 from Base:System

Switch to threaded zstd compression with a pool of 8 threads

OBS-URL: https://build.opensuse.org/request/show/928134
OBS-URL: https://build.opensuse.org/package/show/openSUSE:Factory/rpm?expand=0&rev=292

rpm.changes

@@ -1,3 +1,11 @@
+-------------------------------------------------------------------
+Thu Oct 14 18:16:22 CEST 2021 - mls@suse.de
+
+- Add support for using a thread pool for threaded zstd compression
+  new patch: zstdpool.diff
+- Switch to threaded zstd compression with a pool of 8 threads
+  new patch: zstdthreaded.diff
+
 -------------------------------------------------------------------
 Thu Sep 23 20:57:45 CEST 2021 - mls@suse.de
 

rpm.spec

@@ -115,6 +115,8 @@ Patch122:       db_conversion.diff
 Patch123:       nextiteratorheaderblob.diff
 Patch131:       posttrans.diff
 Patch132:       verbosearg.diff
+Patch133:       zstdpool.diff
+Patch134:       zstdthreaded.diff
 Patch200:       finddebuginfo.diff
 Patch201:       finddebuginfo-absolute-links.diff
 Patch202:       debugsubpkg.diff
@@ -242,7 +244,7 @@ rm -rf sqlite
 %patch                   -P 93 -P 94                         -P 99
 %patch -P 100        -P 102 -P 103
 %patch                                                  -P 117
-%patch -P 122 -P 123               -P 131 -P 132
+%patch -P 122 -P 123               -P 131 -P 132 -P 133 -P 134
 
 # debugedit patches
 pushd debugedit-5.0

zstdpool.diff

@@ -0,0 +1,63 @@
+--- ./rpmio/rpmio.c.orig	2021-10-14 12:34:22.865316722 +0000
++++ ./rpmio/rpmio.c	2021-10-14 13:37:56.835119919 +0000
+@@ -8,6 +8,7 @@
+ #include <ctype.h>
+ #include <dirent.h>
+ #include <fcntl.h>
++#include <pthread.h>
+ #if defined(__linux__)
+ #include <sys/personality.h>
+ #endif
+@@ -1034,6 +1035,7 @@ static const FDIO_t lzdio = &lzdio_s;
+ /* Support for ZSTD library.  */
+ #ifdef HAVE_ZSTD
+ 
++#define ZSTD_STATIC_LINKING_ONLY
+ #include <zstd.h>
+ 
+ typedef struct rpmzstd_s {
+@@ -1048,6 +1050,27 @@ typedef struct rpmzstd_s {
+     ZSTD_outBuffer zob;         /*!< ZSTD_outBuffer */
+ } * rpmzstd;
+ 
++#if ZSTD_VERSION_NUMBER >= 10407
++
++static pthread_once_t zstdThreadPoolCreated = PTHREAD_ONCE_INIT;
++static ZSTD_threadPool *zstdThreadPool;
++
++static void zstdCreateThreadPool(void)
++{
++    int numthreads = rpmExpandNumeric("%{?_zstd_pool_threads}%{?!_zstd_pool_threads:-1}");
++    if (numthreads >= 0)
++        numthreads = get_compression_threads(numthreads > 0 ? numthreads : -1);
++    if (numthreads > 0) {
++        zstdThreadPool = ZSTD_createThreadPool(numthreads);
++        if (!zstdThreadPool)
++            rpmlog(RPMLOG_WARNING, "Could not create zstd thread pool for %d threads\n", numthreads);
++        else
++            rpmlog(RPMLOG_DEBUG, "Created zstd thread pool for %d threads\n", numthreads);
++    }
++}
++
++#endif
++
+ static rpmzstd rpmzstdNew(int fdno, const char *fmode)
+ {
+     int flags = 0;
+@@ -1133,8 +1156,15 @@ static rpmzstd rpmzstdNew(int fdno, cons
+ 
+ 	threads = get_compression_threads(threads);
+ 	if (threads > 0) {
+-	    if (ZSTD_isError (ZSTD_CCtx_setParameter(_stream, ZSTD_c_nbWorkers, threads)))
++	    if (ZSTD_isError (ZSTD_CCtx_setParameter(_stream, ZSTD_c_nbWorkers, threads))) {
+ 		rpmlog(RPMLOG_DEBUG, "zstd library does not support multi-threading\n");
++	    } else {
++#if ZSTD_VERSION_NUMBER >= 10407
++		pthread_once(&zstdThreadPoolCreated, zstdCreateThreadPool);
++		if (zstdThreadPool)
++		    ZSTD_CCtx_refThreadPool(_stream, zstdThreadPool);
++#endif
++	    }
+ 	}
+ 
+ 	nb = ZSTD_CStreamOutSize();

zstdthreaded.diff

@@ -0,0 +1,14 @@
+--- macros.in.orig	2021-10-15 07:40:15.601628187 +0000
++++ macros.in	2021-10-15 07:42:23.557282503 +0000
+@@ -374,7 +374,10 @@ package or when debugging this package.\
+ #		"w.ufdio"	uncompressed
+ #
+ #%_source_payload	w9.gzdio
+-%_binary_payload	w19.zstdio
++%_binary_payload	w19T0.zstdio
++
++# use a pool with 8 threads for threaded zstd compression
++%_zstd_pool_threads	8
+ 
+ #	Algorithm to use for generating file checksum digests on build.
+ #	If not specified or 0, MD5 is used.