testing/rpm
SHA256
3
0
Fork 0
forked from pool/rpm
Code Pull Requests Activity

Accepting request 928134 from Base:System

Browse Source 
Switch to threaded zstd compression with a pool of 8 threads

OBS-URL: https://build.opensuse.org/request/show/928134
OBS-URL: https://build.opensuse.org/package/show/openSUSE:Factory/rpm?expand=0&rev=292
This commit is contained in:
Dominique Leuenberger 2021-11-03 16:25:09 +00:00 committed by Git OBS Bridge
commit 0d2ad6c39d
4 changed files with 88 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
rpm.changes
View File

@ -1,3 +1,11 @@
-------------------------------------------------------------------
Thu Oct 14 18:16:22 CEST 2021 - mls@suse.de
- Add support for using a thread pool for threaded zstd compression
new patch: zstdpool.diff
- Switch to threaded zstd compression with a pool of 8 threads
new patch: zstdthreaded.diff
-------------------------------------------------------------------
Thu Sep 23 20:57:45 CEST 2021 - mls@suse.de

4
rpm.spec
View File

@ -115,6 +115,8 @@ Patch122: db_conversion.diff
Patch123: nextiteratorheaderblob.diff
Patch131: posttrans.diff
Patch132: verbosearg.diff
Patch133: zstdpool.diff
Patch134: zstdthreaded.diff
Patch200: finddebuginfo.diff
Patch201: finddebuginfo-absolute-links.diff
Patch202: debugsubpkg.diff
@ -242,7 +244,7 @@ rm -rf sqlite
%patch -P 93 -P 94 -P 99
%patch -P 100 -P 102 -P 103
%patch -P 117
%patch -P 122 -P 123 -P 131 -P 132
%patch -P 122 -P 123 -P 131 -P 132 -P 133 -P 134
# debugedit patches
pushd debugedit-5.0

63
zstdpool.diff Normal file
View File

@ -0,0 +1,63 @@
--- ./rpmio/rpmio.c.orig 2021-10-14 12:34:22.865316722 +0000
+++ ./rpmio/rpmio.c 2021-10-14 13:37:56.835119919 +0000
@@ -8,6 +8,7 @@
#include <ctype.h>
#include <dirent.h>
#include <fcntl.h>
+#include <pthread.h>
#if defined(__linux__)
#include <sys/personality.h>
#endif
@@ -1034,6 +1035,7 @@ static const FDIO_t lzdio = &lzdio_s;
/* Support for ZSTD library. */
#ifdef HAVE_ZSTD
+#define ZSTD_STATIC_LINKING_ONLY
#include <zstd.h>
typedef struct rpmzstd_s {
@@ -1048,6 +1050,27 @@ typedef struct rpmzstd_s {
ZSTD_outBuffer zob; /*!< ZSTD_outBuffer */
} * rpmzstd;
+#if ZSTD_VERSION_NUMBER >= 10407
+
+static pthread_once_t zstdThreadPoolCreated = PTHREAD_ONCE_INIT;
+static ZSTD_threadPool *zstdThreadPool;
+
+static void zstdCreateThreadPool(void)
+{
+ int numthreads = rpmExpandNumeric("%{?_zstd_pool_threads}%{?!_zstd_pool_threads:-1}");
+ if (numthreads >= 0)
+ numthreads = get_compression_threads(numthreads > 0 ? numthreads : -1);
+ if (numthreads > 0) {
+ zstdThreadPool = ZSTD_createThreadPool(numthreads);
+ if (!zstdThreadPool)
+ rpmlog(RPMLOG_WARNING, "Could not create zstd thread pool for %d threads\n", numthreads);
+ else
+ rpmlog(RPMLOG_DEBUG, "Created zstd thread pool for %d threads\n", numthreads);
+ }
+}
+
+#endif
+
static rpmzstd rpmzstdNew(int fdno, const char *fmode)
{
int flags = 0;
@@ -1133,8 +1156,15 @@ static rpmzstd rpmzstdNew(int fdno, cons
threads = get_compression_threads(threads);
if (threads > 0) {
- if (ZSTD_isError (ZSTD_CCtx_setParameter(_stream, ZSTD_c_nbWorkers, threads)))
+ if (ZSTD_isError (ZSTD_CCtx_setParameter(_stream, ZSTD_c_nbWorkers, threads))) {
rpmlog(RPMLOG_DEBUG, "zstd library does not support multi-threading\n");
+ } else {
+#if ZSTD_VERSION_NUMBER >= 10407
+ pthread_once(&zstdThreadPoolCreated, zstdCreateThreadPool);
+ if (zstdThreadPool)
+ ZSTD_CCtx_refThreadPool(_stream, zstdThreadPool);
+#endif
+ }
}
nb = ZSTD_CStreamOutSize();

14
zstdthreaded.diff Normal file
View File

@ -0,0 +1,14 @@
--- macros.in.orig 2021-10-15 07:40:15.601628187 +0000
+++ macros.in 2021-10-15 07:42:23.557282503 +0000
@@ -374,7 +374,10 @@ package or when debugging this package.\
# "w.ufdio" uncompressed
#
#%_source_payload w9.gzdio
-%_binary_payload w19.zstdio
+%_binary_payload w19T0.zstdio
+
+# use a pool with 8 threads for threaded zstd compression
+%_zstd_pool_threads 8
# Algorithm to use for generating file checksum digests on build.
# If not specified or 0, MD5 is used.