From 8f9aecea69f75bd02f4bff42b1e918028f075a8791e74e76222888559759ea6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Schr=C3=B6der?= Date: Thu, 14 Oct 2021 16:17:41 +0000 Subject: [PATCH 1/2] - Add support for using a thread pool for threaded zstd compression new patch: zstdpool.diff OBS-URL: https://build.opensuse.org/package/show/Base:System/rpm?expand=0&rev=591 --- rpm.changes | 6 +++++ rpm.spec | 3 ++- zstdpool.diff | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 zstdpool.diff diff --git a/rpm.changes b/rpm.changes index f75ed2b..b8af7bd 100644 --- a/rpm.changes +++ b/rpm.changes @@ -1,3 +1,9 @@ +------------------------------------------------------------------- +Thu Oct 14 18:16:22 CEST 2021 - mls@suse.de + +- Add support for using a thread pool for threaded zstd compression + new patch: zstdpool.diff + ------------------------------------------------------------------- Thu Sep 23 20:57:45 CEST 2021 - mls@suse.de diff --git a/rpm.spec b/rpm.spec index 1fca70c..bbf0552 100644 --- a/rpm.spec +++ b/rpm.spec @@ -115,6 +115,7 @@ Patch122: db_conversion.diff Patch123: nextiteratorheaderblob.diff Patch131: posttrans.diff Patch132: verbosearg.diff +Patch133: zstdpool.diff Patch200: finddebuginfo.diff Patch201: finddebuginfo-absolute-links.diff Patch202: debugsubpkg.diff @@ -242,7 +243,7 @@ rm -rf sqlite %patch -P 93 -P 94 -P 99 %patch -P 100 -P 102 -P 103 %patch -P 117 -%patch -P 122 -P 123 -P 131 -P 132 +%patch -P 122 -P 123 -P 131 -P 132 -P 133 # debugedit patches pushd debugedit-5.0 diff --git a/zstdpool.diff b/zstdpool.diff new file mode 100644 index 0000000..a66e7ac --- /dev/null +++ b/zstdpool.diff @@ -0,0 +1,63 @@ +--- ./rpmio/rpmio.c.orig 2021-10-14 12:34:22.865316722 +0000 ++++ ./rpmio/rpmio.c 2021-10-14 13:37:56.835119919 +0000 +@@ -8,6 +8,7 @@ + #include + #include + #include ++#include + #if defined(__linux__) + #include + #endif +@@ -1034,6 +1035,7 @@ static const FDIO_t lzdio = &lzdio_s; + /* Support for ZSTD library. */ + #ifdef HAVE_ZSTD + ++#define ZSTD_STATIC_LINKING_ONLY + #include + + typedef struct rpmzstd_s { +@@ -1048,6 +1050,27 @@ typedef struct rpmzstd_s { + ZSTD_outBuffer zob; /*!< ZSTD_outBuffer */ + } * rpmzstd; + ++#if ZSTD_VERSION_NUMBER >= 10407 ++ ++static pthread_once_t zstdThreadPoolCreated = PTHREAD_ONCE_INIT; ++static ZSTD_threadPool *zstdThreadPool; ++ ++static void zstdCreateThreadPool(void) ++{ ++ int numthreads = rpmExpandNumeric("%{?_zstd_pool_threads}%{?!_zstd_pool_threads:-1}"); ++ if (numthreads >= 0) ++ numthreads = get_compression_threads(numthreads > 0 ? numthreads : -1); ++ if (numthreads > 0) { ++ zstdThreadPool = ZSTD_createThreadPool(numthreads); ++ if (!zstdThreadPool) ++ rpmlog(RPMLOG_WARNING, "Could not create zstd thread pool for %d threads\n", numthreads); ++ else ++ rpmlog(RPMLOG_DEBUG, "Created zstd thread pool for %d threads\n", numthreads); ++ } ++} ++ ++#endif ++ + static rpmzstd rpmzstdNew(int fdno, const char *fmode) + { + int flags = 0; +@@ -1133,8 +1156,15 @@ static rpmzstd rpmzstdNew(int fdno, cons + + threads = get_compression_threads(threads); + if (threads > 0) { +- if (ZSTD_isError (ZSTD_CCtx_setParameter(_stream, ZSTD_c_nbWorkers, threads))) ++ if (ZSTD_isError (ZSTD_CCtx_setParameter(_stream, ZSTD_c_nbWorkers, threads))) { + rpmlog(RPMLOG_DEBUG, "zstd library does not support multi-threading\n"); ++ } else { ++#if ZSTD_VERSION_NUMBER >= 10407 ++ pthread_once(&zstdThreadPoolCreated, zstdCreateThreadPool); ++ if (zstdThreadPool) ++ ZSTD_CCtx_refThreadPool(_stream, zstdThreadPool); ++#endif ++ } + } + + nb = ZSTD_CStreamOutSize(); From b3fdb49a444a60e5b2977d1d0fd084a746bf6736b93da0607c403194b216a672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Schr=C3=B6der?= Date: Fri, 15 Oct 2021 07:44:17 +0000 Subject: [PATCH 2/2] - Switch to threaded zstd compression with a pool of 8 threads new patch: zstdthreaded.diff OBS-URL: https://build.opensuse.org/package/show/Base:System/rpm?expand=0&rev=592 --- rpm.changes | 2 ++ rpm.spec | 3 ++- zstdthreaded.diff | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 zstdthreaded.diff diff --git a/rpm.changes b/rpm.changes index b8af7bd..1b8734e 100644 --- a/rpm.changes +++ b/rpm.changes @@ -3,6 +3,8 @@ Thu Oct 14 18:16:22 CEST 2021 - mls@suse.de - Add support for using a thread pool for threaded zstd compression new patch: zstdpool.diff +- Switch to threaded zstd compression with a pool of 8 threads + new patch: zstdthreaded.diff ------------------------------------------------------------------- Thu Sep 23 20:57:45 CEST 2021 - mls@suse.de diff --git a/rpm.spec b/rpm.spec index bbf0552..2d18fe5 100644 --- a/rpm.spec +++ b/rpm.spec @@ -116,6 +116,7 @@ Patch123: nextiteratorheaderblob.diff Patch131: posttrans.diff Patch132: verbosearg.diff Patch133: zstdpool.diff +Patch134: zstdthreaded.diff Patch200: finddebuginfo.diff Patch201: finddebuginfo-absolute-links.diff Patch202: debugsubpkg.diff @@ -243,7 +244,7 @@ rm -rf sqlite %patch -P 93 -P 94 -P 99 %patch -P 100 -P 102 -P 103 %patch -P 117 -%patch -P 122 -P 123 -P 131 -P 132 -P 133 +%patch -P 122 -P 123 -P 131 -P 132 -P 133 -P 134 # debugedit patches pushd debugedit-5.0 diff --git a/zstdthreaded.diff b/zstdthreaded.diff new file mode 100644 index 0000000..0b3b991 --- /dev/null +++ b/zstdthreaded.diff @@ -0,0 +1,14 @@ +--- macros.in.orig 2021-10-15 07:40:15.601628187 +0000 ++++ macros.in 2021-10-15 07:42:23.557282503 +0000 +@@ -374,7 +374,10 @@ package or when debugging this package.\ + # "w.ufdio" uncompressed + # + #%_source_payload w9.gzdio +-%_binary_payload w19.zstdio ++%_binary_payload w19T0.zstdio ++ ++# use a pool with 8 threads for threaded zstd compression ++%_zstd_pool_threads 8 + + # Algorithm to use for generating file checksum digests on build. + # If not specified or 0, MD5 is used.