Accepting request 1266178 from science:HPC

OBS-URL: https://build.opensuse.org/request/show/1266178
OBS-URL: https://build.opensuse.org/package/show/openSUSE:Factory/openucx?expand=0&rev=34

UCT-IB-UD-Use-GRH-to-detect-address-family-on-non-Mellanox-hardware.patch

@@ -0,0 +1,224 @@
+commit d437b65a6df080416048067141b1c206a52bdc78
+Author: Nathan Hjelm <hjelmn@google.com>
+Date:   Wed Oct 16 20:32:48 2024 +0000
+
+    UCT/IB/UD: Use GRH to detect address family on non-Mellanox hardware
+    
+    Setting the service level in the work completion is a Mellanox-specific feature,
+    so it can not be relied on to detect IPv4 vs IPv6. This commit fixes the
+    detection logic for non-Mellanox providers by detecting the address class from
+    the grh instead. This is done by detecting either 0x6a (IPv6) at offset 0 or
+    0x45 (IPv4) at offset 20 of the receive buffer. Since the first 20B of IPv4
+    packets are undefined ud_verbs sets the first byte of each posted receive to a
+    known value (0xff) since the provider is unliklely to touch these bytes. This
+    commit makes no changes to the mlx5 code which continues to rely on the CQE data
+    to determine if a packet is IPv4 or IPv6. It can be updated to use the non-mlx5
+    logic but since the IP version is present in the CGE there is no need.
+    
+    Signed-off-by: Nathan Hjelm <hjelmn@google.com>
+
+diff --git src/uct/ib/mlx5/ib_mlx5.h src/uct/ib/mlx5/ib_mlx5.h
+index 3183ea460a8a..3ec48b7197d8 100644
+--- src/uct/ib/mlx5/ib_mlx5.h
++++ src/uct/ib/mlx5/ib_mlx5.h
+@@ -1,6 +1,7 @@
+ /**
+ * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2014. ALL RIGHTS RESERVED.
+ * Copyright (C) ARM Ltd. 2016.  ALL RIGHTS RESERVED.
++* Copyright (c) Google, LLC, 2024. ALL RIGHTS RESERVED.
+ *
+ * See file LICENSE for terms.
+ */
+@@ -66,6 +67,9 @@
+ #define UCT_IB_MLX5_ATOMIC_MODE_EXT      3
+ #define UCT_IB_MLX5_CQE_FLAG_L3_IN_DATA  UCS_BIT(28) /* GRH/IP in the receive buffer */
+ #define UCT_IB_MLX5_CQE_FLAG_L3_IN_CQE   UCS_BIT(29) /* GRH/IP in the CQE */
++/* Bits 24-26 of flags_rqpn indicate the packet type */
++#define UCT_IB_MLX5_RQPN_ROCE_FLAG_IPV6  UCS_BIT(24)
++#define UCT_IB_MLX5_RQPN_ROCE_FLAG_IPV4  UCS_BIT(25)
+ #define UCT_IB_MLX5_CQE_FORMAT_MASK      0xc
+ #define UCT_IB_MLX5_MINICQE_ARR_MAX_SIZE 7
+ #define UCT_IB_MLX5_MP_RQ_BYTE_CNT_MASK  0x0000FFFF  /* Byte count mask for multi-packet RQs */
+diff --git src/uct/ib/mlx5/ib_mlx5.inl src/uct/ib/mlx5/ib_mlx5.inl
+index 6602143c8bf5..2aa58455d5cd 100644
+--- src/uct/ib/mlx5/ib_mlx5.inl
++++ src/uct/ib/mlx5/ib_mlx5.inl
+@@ -1,5 +1,6 @@
+ /**
+  * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2016. ALL RIGHTS RESERVED.
++ * Copyright (c) Google, LLC, 2024. ALL RIGHTS RESERVED.
+  *
+  * See file LICENSE for terms.
+  */
+@@ -88,6 +89,35 @@ uct_ib_mlx5_cqe_is_grh_present(struct mlx5_cqe64* cqe)
+                                    UCT_IB_MLX5_CQE_FLAG_L3_IN_CQE);
+ }
+ 
++static UCS_F_ALWAYS_INLINE size_t
++uct_ib_mlx5_cqe_roce_gid_len(struct mlx5_cqe64* cqe)
++{
++    /*
++     * Take the packet type from CQE, because:
++     * 1. According to Annex17_RoCEv2 (A17.4.5.1):
++     * For UD, the Completion Queue Entry (CQE) includes remote address
++     * information (InfiniBand Specification Vol. 1 Rev 1.2.1 Section 11.4.2.1).
++     * For RoCEv2, the remote address information comprises the source L2
++     * Address and a flag that indicates if the received frame is an IPv4,
++     * IPv6 or RoCE packet.
++     *
++     * 2. According to PRM, for responder UD/DC over RoCE sl represents RoCE
++     * packet type as:
++     * bit 3    : when set R-RoCE frame contains an UDP header otherwise not
++     * Bits[2:0]: L3_Header_Type, as defined below
++     *     - 0x0 : GRH - (RoCE v1.0)
++     *     - 0x1 : IPv6 - (RoCE v1.5/v2.0)
++     *     - 0x2 : IPv4 - (RoCE v1.5/v2.0)
++     *
++     * The service level is the most significant byte of cqe->flags_rqpn.
++     *
++     * Alternatively, this could be detected by examining the packet contents
++     * as is done for non-mlx5 transports.
++     */
++    return (cqe->flags_rqpn & htonl(UCT_IB_MLX5_RQPN_ROCE_FLAG_IPV4)) ?
++        UCS_IPV4_ADDR_LEN : UCS_IPV6_ADDR_LEN;
++}
++
+ static UCS_F_ALWAYS_INLINE void*
+ uct_ib_mlx5_gid_from_cqe(struct mlx5_cqe64* cqe)
+ {
+diff --git src/uct/ib/mlx5/ud/ud_mlx5.c src/uct/ib/mlx5/ud/ud_mlx5.c
+index 58f4ae6446a3..27a96b1b615b 100644
+--- src/uct/ib/mlx5/ud/ud_mlx5.c
++++ src/uct/ib/mlx5/ud/ud_mlx5.c
+@@ -2,6 +2,7 @@
+ * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2019. ALL RIGHTS RESERVED.
+ * Copyright (C) ARM Ltd. 2017.  ALL RIGHTS RESERVED.
+ * Copyright (C) Advanced Micro Devices, Inc. 2024. ALL RIGHTS RESERVED.
++* Copyright (c) Google, LLC, 2024. ALL RIGHTS RESERVED.
+ *
+ * See file LICENSE for terms.
+ */
+@@ -521,7 +522,7 @@ uct_ud_mlx5_iface_poll_rx(uct_ud_mlx5_iface_t *iface, int is_async)
+ 
+     if (!uct_ud_iface_check_grh(&iface->super, packet,
+                                 uct_ib_mlx5_cqe_is_grh_present(cqe),
+-                                cqe->flags_rqpn & 0xFF)) {
++                                uct_ib_mlx5_cqe_roce_gid_len(cqe))) {
+         ucs_mpool_put_inline(desc);
+         goto out_polled;
+     }
+diff --git src/uct/ib/ud/base/ud_iface.h src/uct/ib/ud/base/ud_iface.h
+index 1efecd291d98..89fa7e3810fc 100644
+--- src/uct/ib/ud/base/ud_iface.h
++++ src/uct/ib/ud/base/ud_iface.h
+@@ -1,5 +1,6 @@
+ /**
+ * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2020. ALL RIGHTS RESERVED.
++* Copyright (c) Google, LLC, 2024. ALL RIGHTS RESERVED.
+ *
+ * See file LICENSE for terms.
+ */
+@@ -395,10 +396,9 @@ static UCS_F_ALWAYS_INLINE void uct_ud_leave(uct_ud_iface_t *iface)
+ 
+ static UCS_F_ALWAYS_INLINE int
+ uct_ud_iface_check_grh(uct_ud_iface_t *iface, void *packet, int is_grh_present,
+-                       uint8_t roce_pkt_type)
++                       size_t gid_len)
+ {
+     struct ibv_grh *grh = (struct ibv_grh *)packet;
+-    size_t gid_len;
+     union ibv_gid *gid;
+     khiter_t khiter;
+     char gid_str[128] UCS_V_UNUSED;
+@@ -412,25 +412,6 @@ uct_ud_iface_check_grh(uct_ud_iface_t *iface, void *packet, int is_grh_present,
+         return 1;
+     }
+ 
+-    /*
+-     * Take the packet type from CQE, because:
+-     * 1. According to Annex17_RoCEv2 (A17.4.5.1):
+-     * For UD, the Completion Queue Entry (CQE) includes remote address
+-     * information (InfiniBand Specification Vol. 1 Rev 1.2.1 Section 11.4.2.1).
+-     * For RoCEv2, the remote address information comprises the source L2
+-     * Address and a flag that indicates if the received frame is an IPv4,
+-     * IPv6 or RoCE packet.
+-     * 2. According to PRM, for responder UD/DC over RoCE sl represents RoCE
+-     * packet type as:
+-     * bit 3    : when set R-RoCE frame contains an UDP header otherwise not
+-     * Bits[2:0]: L3_Header_Type, as defined below
+-     *     - 0x0 : GRH - (RoCE v1.0)
+-     *     - 0x1 : IPv6 - (RoCE v1.5/v2.0)
+-     *     - 0x2 : IPv4 - (RoCE v1.5/v2.0)
+-     */
+-    gid_len = ((roce_pkt_type & UCT_IB_CQE_SL_PKTYPE_MASK) == 0x2) ?
+-              UCS_IPV4_ADDR_LEN : UCS_IPV6_ADDR_LEN;
+-
+     if (ucs_likely((gid_len == iface->gid_table.last_len) &&
+                     uct_ud_gid_equal(&grh->dgid, &iface->gid_table.last,
+                                      gid_len))) {
+diff --git src/uct/ib/ud/verbs/ud_verbs.c src/uct/ib/ud/verbs/ud_verbs.c
+index 989bdb59d08f..848dc4e5cd66 100644
+--- src/uct/ib/ud/verbs/ud_verbs.c
++++ src/uct/ib/ud/verbs/ud_verbs.c
+@@ -1,5 +1,6 @@
+ /**
+ * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2019. ALL RIGHTS RESERVED.
++* Copyright (c) Google, LLC, 2024. ALL RIGHTS RESERVED.
+ *
+ * See file LICENSE for terms.
+ */
+@@ -393,6 +394,20 @@ uct_ud_verbs_iface_poll_tx(uct_ud_verbs_iface_t *iface, int is_async)
+     return 1;
+ }
+ 
++static UCS_F_ALWAYS_INLINE size_t uct_ud_verbs_iface_get_gid_len(void *packet)
++{
++  /* The GRH will contain either an IPv4 or IPv6 header. If the former is
++   * present the header will start at offset 20 in the buffer otherwise it
++   * will start at offset 0. Since the two headers are of fixed size (20 or
++   * 40 bytes) this means we will either see 0x6? at offset 0 (IPv6) or 0x45
++   * at offset 20. The detection is a little tricky for IPv6 given that the
++   * first 20B are undefined for IPv4. To overcome this the first byte of
++   * the posted receive buffer is set to 0xff.
++   */
++  return ((((uint8_t*)packet)[0] & 0xf0) == 0x60) ? UCS_IPV6_ADDR_LEN :
++                                                    UCS_IPV4_ADDR_LEN;
++}
++
+ static UCS_F_ALWAYS_INLINE unsigned
+ uct_ud_verbs_iface_poll_rx(uct_ud_verbs_iface_t *iface, int is_async)
+ {
+@@ -413,7 +428,8 @@ uct_ud_verbs_iface_poll_rx(uct_ud_verbs_iface_t *iface, int is_async)
+ 
+     UCT_IB_IFACE_VERBS_FOREACH_RXWQE(&iface->super.super, i, packet, wc, num_wcs) {
+         if (!uct_ud_iface_check_grh(&iface->super, packet,
+-                                    wc[i].wc_flags & IBV_WC_GRH, wc[i].sl)) {
++                                    wc[i].wc_flags & IBV_WC_GRH,
++                                    uct_ud_verbs_iface_get_gid_len(packet))) {
+             ucs_mpool_put_inline((void*)wc[i].wr_id);
+             continue;
+         }
+@@ -696,7 +712,7 @@ uct_ud_verbs_iface_post_recv_always(uct_ud_verbs_iface_t *iface, int max)
+     struct ibv_recv_wr *bad_wr;
+     uct_ib_recv_wr_t *wrs;
+     unsigned count;
+-    int ret;
++    int ret, i;
+ 
+     wrs  = ucs_alloca(sizeof *wrs  * max);
+ 
+@@ -706,6 +722,14 @@ uct_ud_verbs_iface_post_recv_always(uct_ud_verbs_iface_t *iface, int max)
+         return;
+     }
+ 
++    /* Set the first byte in the receive buffer grh to a known value not equal to
++     * 0x6?. This should aid in the detection of IPv6 vs IPv4 because the first
++     * byte is undefined in the later and 0x6? in the former. It is unlikely
++     * this byte is touched with IPv4. */
++    for (i = 0; i < count; ++i) {
++        ((uint8_t*)wrs[i].sg.addr)[0] = 0xff;
++    }
++
+     ret = ibv_post_recv(iface->super.qp, &wrs[0].ibwr, &bad_wr);
+     if (ret != 0) {
+         ucs_fatal("ibv_post_recv() returned %d: %m", ret);

openucx-s390x-support.patch

@@ -1,6 +1,6 @@
-commit 328a69d07b618e0aa83fe2351e8d7ca4fc1b2f00
+commit ba1d7048df80ee535e01335992f70568e2f88c80
 Author: Nicolas Morey <nmorey@suse.com>
-Date:   Mon Feb 13 17:04:14 2023 +0100
+Date:   Wed Feb 19 16:46:33 2025 +0100
 
     openucx s390x support
     
@@ -32,42 +32,26 @@ index e5e66266d695..ef7e4ede93ce 100644
  
  AS_IF([test "x$bistro_hooks_happy" = "xyes"],
        [AC_DEFINE([UCM_BISTRO_HOOKS], [1], [Enable BISTRO hooks])],
-diff --git src/tools/info/sys_info.c src/tools/info/sys_info.c
-index e5aff871d491..2b7c54319f53 100644
---- src/tools/info/sys_info.c
-+++ src/tools/info/sys_info.c
-@@ -47,7 +47,8 @@ static const char* cpu_vendor_names[] = {
-     [UCS_CPU_VENDOR_GENERIC_ARM]      = "Generic ARM",
-     [UCS_CPU_VENDOR_GENERIC_PPC]      = "Generic PPC",
-     [UCS_CPU_VENDOR_FUJITSU_ARM]      = "Fujitsu ARM",
--    [UCS_CPU_VENDOR_ZHAOXIN]          = "Zhaoxin"
-+    [UCS_CPU_VENDOR_ZHAOXIN]          = "Zhaoxin",
-+    [UCS_CPU_VENDOR_GENERIC_IBM]      = "Generic IBM"
- };
- 
- static double measure_memcpy_bandwidth(size_t size)
 diff --git src/ucm/Makefile.am src/ucm/Makefile.am
-index 48b82bf89cbe..582f83d1ea82 100644
+index 7866aa0ac13b..2d44e20f124d 100644
 --- src/ucm/Makefile.am
 +++ src/ucm/Makefile.am
-@@ -31,7 +31,8 @@ noinst_HEADERS = \
- 	bistro/bistro.h \
- 	bistro/bistro_x86_64.h \
+@@ -35,6 +35,7 @@ noinst_HEADERS = \
  	bistro/bistro_aarch64.h \
--	bistro/bistro_ppc64.h
-+	bistro/bistro_ppc64.h \
+ 	bistro/bistro_ppc64.h \
+ 	bistro/bistro_rv64.h
 +	bistro/bistro_s390x.h
  
  libucm_la_SOURCES = \
  	event/event.c \
 diff --git src/ucm/bistro/bistro.h src/ucm/bistro/bistro.h
-index b622e3c14fbb..4acd9e9cdb83 100644
+index fffbe738b116..31859a84b159 100644
 --- src/ucm/bistro/bistro.h
 +++ src/ucm/bistro/bistro.h
-@@ -20,6 +20,8 @@ typedef struct ucm_bistro_restore_point ucm_bistro_restore_point_t;
- #  include "bistro_aarch64.h"
- #elif defined(__x86_64__)
+@@ -23,6 +23,8 @@ typedef struct ucm_bistro_restore_point ucm_bistro_restore_point_t;
  #  include "bistro_x86_64.h"
+ #elif defined(__riscv)
+ #  include "bistro_rv64.h"
 +#elif defined(__s390x__)
 +#  include "bistro_s390x.h"
  #else
@@ -75,10 +59,10 @@ index b622e3c14fbb..4acd9e9cdb83 100644
  #endif
 diff --git src/ucm/bistro/bistro_s390x.h src/ucm/bistro/bistro_s390x.h
 new file mode 100644
-index 000000000000..c0f427f4984a
+index 000000000000..2beb5de54fab
 --- /dev/null
 +++ src/ucm/bistro/bistro_s390x.h
-@@ -0,0 +1,18 @@
+@@ -0,0 +1,27 @@
 +#ifndef UCM_BISTRO_BISTRO_S390X_H_
 +#define UCM_BISTRO_BISTRO_S390X_H_
 +
@@ -90,55 +74,65 @@ index 000000000000..c0f427f4984a
 +#define UCM_BISTRO_PROLOGUE
 +#define UCM_BISTRO_EPILOGUE
 +
++typedef struct ucm_bistro_patch {
++} UCS_S_PACKED ucm_bistro_patch_t;
++typedef struct {
++} UCS_S_PACKED ucm_bistro_lock_t;
++
 +static inline ucs_status_t ucm_bistro_patch(void *func_ptr, void *hook, const char *symbol,
-+											void **orig_func_p,
-+											ucm_bistro_restore_point_t **rp){
++					    void **orig_func_p,
++					    ucm_bistro_restore_point_t **rp){
 +	return UCS_ERR_UNSUPPORTED;
 +}
 +
++static inline void ucm_bistro_patch_lock(void * UCS_V_UNUSED dst)
++{
++}
++
 +#endif
 diff --git src/ucs/Makefile.am src/ucs/Makefile.am
-index c7696d56f25d..c63b32bad844 100644
+index 86a469a60bcc..6751bad764b8 100644
 --- src/ucs/Makefile.am
 +++ src/ucs/Makefile.am
-@@ -22,6 +22,7 @@ libucs_la_LIBADD   = $(LIBM) $(top_builddir)/src/ucm/libucm.la $(BFD_LIBS)
- nobase_dist_libucs_la_HEADERS = \
+@@ -24,6 +24,7 @@ nobase_dist_libucs_la_HEADERS = \
  	arch/aarch64/bitops.h \
  	arch/ppc64/bitops.h \
+ 	arch/rv64/bitops.h \
 +	arch/s390x/bitops.h \
  	arch/x86_64/bitops.h \
  	arch/bitops.h \
  	algorithm/crc.h \
-@@ -82,12 +83,14 @@ nobase_dist_libucs_la_HEADERS = \
- 	arch/aarch64/global_opts.h \
+@@ -87,6 +88,7 @@ nobase_dist_libucs_la_HEADERS = \
  	arch/generic/atomic.h \
  	arch/ppc64/global_opts.h \
+ 	arch/rv64/global_opts.h \
 +	arch/s390x/global_opts.h \
  	arch/global_opts.h
  
  noinst_HEADERS = \
- 	arch/aarch64/cpu.h \
+@@ -94,6 +96,7 @@ noinst_HEADERS = \
  	arch/generic/cpu.h \
  	arch/ppc64/cpu.h \
+ 	arch/rv64/cpu.h \
 +	arch/s390x/cpu.h \
  	arch/x86_64/cpu.h \
  	arch/cpu.h \
  	config/ucm_opts.h \
-@@ -138,6 +141,7 @@ libucs_la_SOURCES = \
+@@ -149,6 +152,7 @@ libucs_la_SOURCES = \
  	algorithm/string_distance.c \
  	arch/aarch64/cpu.c \
  	arch/aarch64/global_opts.c \
 +	arch/s390x/global_opts.c \
  	arch/ppc64/timebase.c \
  	arch/ppc64/global_opts.c \
- 	arch/x86_64/cpu.c \
+ 	arch/rv64/cpu.c \
 diff --git src/ucs/arch/atomic.h src/ucs/arch/atomic.h
-index 52be711c1d0a..8f1d62a28dc9 100644
+index 849647902fab..a328c37e2020 100644
 --- src/ucs/arch/atomic.h
 +++ src/ucs/arch/atomic.h
-@@ -15,6 +15,8 @@
+@@ -18,6 +18,8 @@
  #  include "generic/atomic.h"
- #elif defined(__aarch64__)
+ #elif defined(__riscv)
  #  include "generic/atomic.h"
 +#elif defined(__s390x__)
 +#  include "generic/atomic.h"
@@ -146,23 +140,23 @@ index 52be711c1d0a..8f1d62a28dc9 100644
  #  error "Unsupported architecture"
  #endif
 diff --git src/ucs/arch/bitops.h src/ucs/arch/bitops.h
-index e89a37d0b673..dd2b9d5b6bcb 100644
+index f8e51c45888a..476631d95eb6 100644
 --- src/ucs/arch/bitops.h
 +++ src/ucs/arch/bitops.h
-@@ -20,6 +20,8 @@ BEGIN_C_DECLS
- #  include "ppc64/bitops.h"
- #elif defined(__aarch64__)
+@@ -23,6 +23,8 @@ BEGIN_C_DECLS
  #  include "aarch64/bitops.h"
+ #elif defined(__riscv)
+ #  include "rv64/bitops.h"
 +#elif defined(__s390x__)
 +#  include "s390x/bitops.h"
  #else
  #  error "Unsupported architecture"
  #endif
 diff --git src/ucs/arch/cpu.c src/ucs/arch/cpu.c
-index ece8f7fb82dd..b35b10ad090a 100644
+index 6fe5e31dba31..f92c53f303cd 100644
 --- src/ucs/arch/cpu.c
 +++ src/ucs/arch/cpu.c
-@@ -63,6 +63,10 @@ const ucs_cpu_builtin_memcpy_t ucs_cpu_builtin_memcpy[UCS_CPU_VENDOR_LAST] = {
+@@ -64,6 +64,10 @@ const ucs_cpu_builtin_memcpy_t ucs_cpu_builtin_memcpy[UCS_CPU_VENDOR_LAST] = {
          .min = UCS_MEMUNITS_INF,
          .max = UCS_MEMUNITS_INF
      },
@@ -173,43 +167,67 @@ index ece8f7fb82dd..b35b10ad090a 100644
      [UCS_CPU_VENDOR_FUJITSU_ARM] = {
          .min = UCS_MEMUNITS_INF,
          .max = UCS_MEMUNITS_INF
-@@ -78,6 +82,7 @@ const size_t ucs_cpu_est_bcopy_bw[UCS_CPU_VENDOR_LAST] = {
-     [UCS_CPU_VENDOR_INTEL]       = UCS_CPU_EST_BCOPY_BW_DEFAULT,
-     [UCS_CPU_VENDOR_AMD]         = UCS_CPU_EST_BCOPY_BW_AMD,
-     [UCS_CPU_VENDOR_GENERIC_ARM] = UCS_CPU_EST_BCOPY_BW_DEFAULT,
-+    [UCS_CPU_VENDOR_GENERIC_IBM] = UCS_CPU_EST_BCOPY_BW_DEFAULT,
-     [UCS_CPU_VENDOR_GENERIC_PPC] = UCS_CPU_EST_BCOPY_BW_DEFAULT,
-     [UCS_CPU_VENDOR_FUJITSU_ARM] = UCS_CPU_EST_BCOPY_BW_FUJITSU_ARM,
-     [UCS_CPU_VENDOR_ZHAOXIN]     = UCS_CPU_EST_BCOPY_BW_DEFAULT
+@@ -82,7 +86,6 @@ const ucs_cpu_builtin_memcpy_t ucs_cpu_builtin_memcpy[UCS_CPU_VENDOR_LAST] = {
+     }
+ };
+ 
+-
+ static void ucs_sysfs_get_cache_size()
+ {
+     char type_str[32];  /* Data/Instruction/Unified */
+@@ -167,6 +170,7 @@ const char *ucs_cpu_vendor_name()
+         [UCS_CPU_VENDOR_GENERIC_ARM]   = "Generic ARM",
+         [UCS_CPU_VENDOR_GENERIC_PPC]   = "Generic PPC",
+         [UCS_CPU_VENDOR_GENERIC_RV64G] = "Generic RV64G",
++	[UCS_CPU_VENDOR_GENERIC_IBM]   = "Generic IBM",
+         [UCS_CPU_VENDOR_FUJITSU_ARM]   = "Fujitsu ARM",
+         [UCS_CPU_VENDOR_ZHAOXIN]       = "Zhaoxin",
+         [UCS_CPU_VENDOR_NVIDIA]        = "Nvidia"
+@@ -197,6 +201,7 @@ const char *ucs_cpu_model_name()
+         [UCS_CPU_MODEL_ZHAOXIN_WUDAOKOU]   = "Wudaokou",
+         [UCS_CPU_MODEL_ZHAOXIN_LUJIAZUI]   = "Lujiazui",
+         [UCS_CPU_MODEL_RV64G]              = "RV64G",
++        [UCS_CPU_MODEL_S390X]              = "S390x",
+         [UCS_CPU_MODEL_NVIDIA_GRACE]       = "Grace"
+     };
+ 
 diff --git src/ucs/arch/cpu.h src/ucs/arch/cpu.h
-index eb557d385670..cfd297e24558 100644
+index 857b8b804cf7..89461d52d406 100644
 --- src/ucs/arch/cpu.h
 +++ src/ucs/arch/cpu.h
-@@ -64,6 +64,7 @@ typedef enum ucs_cpu_vendor {
+@@ -41,6 +41,7 @@ typedef enum ucs_cpu_model {
+     UCS_CPU_MODEL_ZHAOXIN_WUDAOKOU,
+     UCS_CPU_MODEL_ZHAOXIN_LUJIAZUI,
+     UCS_CPU_MODEL_RV64G,
++    UCS_CPU_MODEL_S390X,
+     UCS_CPU_MODEL_NVIDIA_GRACE,
+     UCS_CPU_MODEL_LAST
+ } ucs_cpu_model_t;
+@@ -70,6 +71,7 @@ typedef enum ucs_cpu_vendor {
      UCS_CPU_VENDOR_AMD,
      UCS_CPU_VENDOR_GENERIC_ARM,
      UCS_CPU_VENDOR_GENERIC_PPC,
 +    UCS_CPU_VENDOR_GENERIC_IBM,
      UCS_CPU_VENDOR_FUJITSU_ARM,
      UCS_CPU_VENDOR_ZHAOXIN,
-     UCS_CPU_VENDOR_LAST
-@@ -99,6 +100,8 @@ typedef struct ucs_cpu_builtin_memcpy {
- #  include "ppc64/cpu.h"
- #elif defined(__aarch64__)
+     UCS_CPU_VENDOR_GENERIC_RV64G,
+@@ -109,6 +111,8 @@ typedef struct ucs_cpu_builtin_memcpy {
  #  include "aarch64/cpu.h"
+ #elif defined(__riscv)
+ #  include "rv64/cpu.h"
 +#elif defined(__s390x__)
 +#  include "s390x/cpu.h"
  #else
  #  error "Unsupported architecture"
  #endif
 diff --git src/ucs/arch/global_opts.h src/ucs/arch/global_opts.h
-index 75d086177a7f..96c670cb60d3 100644
+index 550d22b8b751..d8e4a7cca694 100644
 --- src/ucs/arch/global_opts.h
 +++ src/ucs/arch/global_opts.h
-@@ -15,6 +15,8 @@
- #  include "ppc64/global_opts.h"
- #elif defined(__aarch64__)
+@@ -18,6 +18,8 @@
  #  include "aarch64/global_opts.h"
+ #elif defined(__riscv)
+ #  include "rv64/global_opts.h"
 +#elif defined(__s390x__)
 +#  include "s390x/global_opts.h"
  #else
@@ -260,10 +278,10 @@ index 000000000000..ce48ff1ff451
 +#endif
 diff --git src/ucs/arch/s390x/cpu.h src/ucs/arch/s390x/cpu.h
 new file mode 100644
-index 000000000000..4f0a87006118
+index 000000000000..e1d41a0ef8b8
 --- /dev/null
 +++ src/ucs/arch/s390x/cpu.h
-@@ -0,0 +1,84 @@
+@@ -0,0 +1,86 @@
 +/**
 +* Copyright (C) Mellanox Technologies Ltd. 2001-2013.  ALL RIGHTS RESERVED.
 +* Copyright (C) ARM Ltd. 2016-2017.  ALL RIGHTS RESERVED.
@@ -290,7 +308,7 @@ index 000000000000..4f0a87006118
 +#define ucs_memory_bus_fence()        asm volatile (""::: "memory")
 +#define ucs_memory_bus_store_fence()  ucs_memory_bus_fence()
 +#define ucs_memory_bus_load_fence()   ucs_memory_bus_fence()
-+#define ucs_memory_bus_wc_flush()     ucs_memory_bus_fence()
++#define ucs_memory_bus_cacheline_wc_flush()     ucs_memory_bus_fence()
 +#define ucs_memory_cpu_fence()        ucs_memory_bus_fence()
 +#define ucs_memory_cpu_store_fence()  ucs_memory_bus_fence()
 +#define ucs_memory_cpu_load_fence()   ucs_memory_bus_fence()
@@ -308,7 +326,7 @@ index 000000000000..4f0a87006118
 +
 +static inline ucs_cpu_model_t ucs_arch_get_cpu_model()
 +{
-+    return UCS_CPU_MODEL_UNKNOWN;
++    return UCS_CPU_MODEL_S390X;
 +}
 +
 +static inline ucs_cpu_vendor_t ucs_arch_get_cpu_vendor()
@@ -329,7 +347,9 @@ index 000000000000..4f0a87006118
 +{
 +}
 +
-+static inline void *ucs_memcpy_relaxed(void *dst, const void *src, size_t len)
++static inline void *ucs_memcpy_relaxed(void *dst, const void *src, size_t len,
++                                       ucs_arch_memcpy_hint_t hint,
++                                       size_t total_len)
 +{
 +    return memcpy(dst, src, len);
 +}
@@ -410,7 +430,7 @@ index 000000000000..225e4e5e896a
 +#endif
 +
 diff --git src/ucs/sys/sys.c src/ucs/sys/sys.c
-index 58e67835c4d0..308f03606d5b 100644
+index d0b5effe11a3..ce22a2097f18 100644
 --- src/ucs/sys/sys.c
 +++ src/ucs/sys/sys.c
 @@ -1258,8 +1258,19 @@ void *ucs_sys_realloc(void *old_ptr, size_t old_length, size_t new_length)

openucx.changes

@@ -1,3 +1,153 @@
+-------------------------------------------------------------------
+Tue Apr  1 12:31:11 UTC 2025 - Nicolas Morey <nicolas.morey@suse.com>
+
+- Add UCT-IB-UD-Use-GRH-to-detect-address-family-on-non-Mellanox-hardware.patch
+  to fix an UD init issue on non-Mellanox RDMA HW (bsc#1240204).
+
+-------------------------------------------------------------------
+Wed Feb 19 15:47:23 UTC 2025 - Nicolas Morey <nicolas.morey@suse.com>
+
+- Update to ucx 1.18.0
+  - UCP
+    - Enabled using CUDA staging buffers for pipeline protocols by default
+    - Added endpoint reconfiguration support for non-reused p2p scenarios
+    - Enabled non-cacheable memory domains, activated for gdr_copy
+    - Added user_data parameter to ucp_ep_query
+    - Added support for host memory pipeline through CUDA buffers for rendezvous protocol
+    - Added global VA infrastructure and memory region in absence of error handling
+    - Made protocol performance node names more informative
+    - Enforced always running on the same thread in single thread mode
+    - Multiple improvements in protocols selection infrastructure
+    - Added UCP_MEM_MAP_LOCK API flag to enforce locked memory mapping
+    - Allowed up-to 64 endpoint lanes for systems with many transports or devices
+    - Added usage tracker to worker
+    - Improved various logging messages
+    - Fixed stack overflow in exported rkey unpack
+    - Removed extra remote-cpu overhead from protocol estimation for zcopy
+    - Fixed performance estimation for rndv pipeline protocols
+    - Fixed ATP sending by picking the correct lane
+    - Fixed missing reg_id on memh creation
+    - Fixed repeated invalidations by retaining existing access flags
+    - Fixed abort reason propagation for rendezvous RTR mtype
+    - Do not check transport availability if it is disabled by UCX_TLS environment variable
+    - Fixed wrong flag being used for checking BCOPY capability
+    - Fixed sending too many ATPs for small messages
+    - Enforced 16 bits size for Active Messages identifiers
+    - Fixed unnecessary status check for emulated AMO
+    - Fixed more than one fragment sending in rendezvous pipeline
+    - Fixed crash by using biggest max frag across all lanes
+    - Fixed missing memory handle flags by copying from parent to child
+    - Fixed worker interface activate count
+    - Fixed flush requests by replacing ATP/flush lane map with lane indexes
+    - Fixed lost uct_flags when merging memory regions
+  - UCT
+    - Fixed memory domain UCT flags description
+  - RDMA CORE (IB, ROCE, etc.)
+    - Added environment variable to manage DC initiator capacity
+    - Added DC dcs_hybrid policy
+    - Reduced MLX5/DV stack size consumption
+    - Added ODP support for verbs and mlx5dv
+    - Added support of CUDA managed memory on IB when ODP is available
+    - Added support of Adaptive Routing on RoCE
+    - Enabled use of implicit ODP with relaxed ordering
+    - Improved GPU-Direct detection in IB transport
+    - Increased DC initiator default count to 32 for performance optimization
+    - Added ConnectX-8 device support with DDP
+    - Added support for subnet filter list for RoCE interfaces
+    - Enhanced the error message to provide more details when a connection cannot be
+      established due to unreachable transports
+    - Added IB MLX5 as a separate UCX module with separate RPM sub-package
+    - Added initial support for GGA transport, for fast DPU memory access
+    - Set IB DevX atomic mode based on device capabilities
+    - Removed DC keepalive mechanism, since the keepalive is done on UCP layer
+    - Optimized cross-gVMI memory registration using indirect memory keys cache
+    - Improved various logging messages
+    - Fixed FETCH_ADD remote access error for ODP/KSM case
+    - Fixed missing conditional compilation checks for DM
+    - Fixed IB MD allocation naming typo
+    - Fixed invalid GIDs filter in IB
+    - Fixed flags usage in MLX5 zcopy_post
+    - Do not limit ODP registration retries
+    - Fixed JUCX failures by considering the number of supported completion vectors
+  - UCS
+    - Added support for wildcards in configuration parameter names
+    - Added ASAN protection to several internal data structures
+    - Reduced stack usage in topology detection code
+    - Improved bitmaps configuration parsing with wider bitfield
+    - Added options to set topology distance between devices
+    - Optimized VFS unix socket watch by using user private folder
+    - Added general IP subnet matching infrastructure
+    - Extend array data structure to support user-provided array copy routine
+    - Improved time units description
+    - Fixed a crash by using heap allocation to process expired timers in batch
+    - Fixed allocation issue on memtrack dump
+    - Fixed deletion of the monitored folder in VFS
+    - Fixed unsafe resize for DC initiator array
+    - Fixed function macro invocation to match C standard
+    - Fixed calling async handler on already released resource
+    - Fixed performance by setting higher bandwidth for different NUMA nodes on Grace
+    - Fixed undeclared value error in timer conversion routine
+    - Fixed uninitialized value access in registration cache
+  - UCM
+    - Extend CUDA memory hooks to include memory mapping APIs
+    - Fixed race condition in parsing proc maps
+    - Fixed mremap failure while parsing /proc/self/maps
+  - TCP
+    - Always bind endpoint to interface
+  - Tools
+    - Improved performance by increasing window size for put_bw and add get_bw in ucx_perftest
+    - Added multi-send flag for receive operations in bandwidth benchmarks in ucx_perftest
+    - Improved ucx_perftest uni-directional test with added fence
+    - Detailed ucx_perftest batch section of command-line documentation
+    - Fixed buffer size potential overflow in ucx_perftest
+    - Fixed missing address when packing memory keys on ucx_perftest
+    - Fixed memory leak for endpoint report in ucx_info
+    - Fixed build without openmp in ucx_perftest
+    - Fixed UCT device override on server side on ucx_perftest
+  - Documentation
+    - Added a section regarding adaptive routing on RoCE
+  - Architecture
+    - Added CPU Model for MI300A
+    - Added Fujitsu ARM specific values to ucx.conf
+    - Added AMD Turin support
+    - Added an optimized non-temporal memory copy implementation for AMD CPU
+  - Build
+    - Improved compiler error reporting with added flag
+    - Improved coverity script to allow faster turnaround time
+    - Improved Intel Compiler detection and support
+    - Fixed using correct ASAN version for running tests
+  - Configuration
+    - Used POSIX bourne syntax to check equality
+    - Fixed build failure by using proper flags in compiler.m4
+    - Fixed perftest MAD support default guessing
+  - GO
+    - Added multi-send flag and user memh support in request params
+    - Added serialized thread mode to avoid subtle races between threads
+    - Fixed make distcheck
+  - Packaging
+    - Improved dpkg-buildpackage sample command by explicitly adding mlx5 related arguments
+- Delete UCS-TIME-Add-math.h-to-provide-INFINITY.patch which was merged upstream
+- Refresh openucx-s390x-support.patch due to API changes
+
+-------------------------------------------------------------------
+Sat Sep  7 14:22:20 UTC 2024 - Nicolas Morey <nicolas.morey@suse.com>
+
+- Refresh openucx-s390x-support.patch to fix compilation on s390x
+
+-------------------------------------------------------------------
+Sat Jun 29 16:55:27 UTC 2024 - Andreas Schwab <schwab@suse.de>
+
+- Enable build on riscv64
+
+-------------------------------------------------------------------
+Wed Jun 26 15:43:05 UTC 2024 - Nicolas Morey <nicolas.morey@suse.com>
+
+- Update to 1.17.0
+  - See NEWS for the complete CHANGELOG
+- Refresh openucx-s390x-support.patch against the latest sources
+- Add upstream fix UCS-TIME-Add-math.h-to-provide-INFINITY.patch
+  to fix compilation on ppc64
+
 -------------------------------------------------------------------
 Mon Feb 26 12:49:43 UTC 2024 - Dominique Leuenberger <dimstar@opensuse.org>
 

openucx.spec

@@ -1,7 +1,7 @@
 #
 # spec file for package openucx
 #
-# Copyright (c) 2023 SUSE LLC
+# Copyright (c) 2025 SUSE LLC
 #
 # All modifications and additions to the file contributed by third parties
 # remain the property of their copyright owners, unless otherwise agreed
@@ -20,7 +20,7 @@
 %define version_suf %{nil}
 
 Name:           openucx
-Version:        1.15.0
+Version:        1.18.0
 Release:        0
 Summary:        Communication layer for Message Passing (MPI)
 License:        BSD-3-Clause
@@ -32,6 +32,7 @@ URL:            http://openucx.org/
 Source:         https://github.com/openucx/ucx/releases/download/v%version%{?version_suf}/ucx-%version.tar.gz
 Patch1:         openucx-s390x-support.patch
 Patch2:         ucm-fix-UCX_MEM_MALLOC_RELOC.patch
+Patch3:         UCT-IB-UD-Use-GRH-to-detect-address-family-on-non-Mellanox-hardware.patch
 BuildRequires:  autoconf >= 2.63
 BuildRequires:  automake >= 1.10
 BuildRequires:  binutils-devel
@@ -48,7 +49,7 @@ BuildRequires:  libtool
 BuildRequires:  pkg-config
 BuildRequires:  zlib-devel
 BuildRoot:      %{_tmppath}/%{name}-%{version}-build
-ExclusiveArch:  aarch64 %power64 x86_64 s390x
+ExclusiveArch:  aarch64 %power64 x86_64 s390x riscv64
 
 %description
 UCX stands for Unified Communication X. UCX provides a communication
@@ -136,10 +137,7 @@ hardware.
 
 %prep
 %setup -qn ucx-%version
-%ifarch s390x
-%patch -P 1
-%endif
-%patch -P 2
+%autopatch -p0
 
 %build
 autoreconf -fi
@@ -160,7 +158,8 @@ export UCX_CFLAGS="$UCX_CFLAGS -mno-sse -mno-sse2"
 		 --disable-debug --disable-assertions \
 		 --disable-params-check \
 		 --with-rc --with-ud --with-dc \
-		 --with-mlx5-dv --with-rdmacm
+                 --with-ib-hw-tm --with-dm --with-devx \
+		 --with-mlx5 --with-rdmacm
 
 # Override BASE_CFLAGS to disable Werror (boo#1121267)
 make %{?_smp_mflags} V=1 BASE_CFLAGS="-g -Wall"
@@ -192,6 +191,8 @@ mv %buildroot/%_bindir/io_demo  %buildroot/%_libexecdir/%{name}/
 %_libdir/pkgconfig/ucx.pc
 %dir %_libdir/cmake/
 %_libdir/cmake/ucx/
+%dir %{_sysconfdir}/ucx/
+%config %{_sysconfdir}/ucx/ucx.conf
 %license LICENSE
 %doc NEWS
 
@@ -230,6 +231,7 @@ mv %buildroot/%_bindir/io_demo  %buildroot/%_libexecdir/%{name}/
 %_libdir/libuct.so.*
 %dir %_libdir/ucx/
 %_libdir/ucx/libuct_*.so.*
+%_libdir/ucx/libucx_perftest_mad.so.*
 
 %files -n libuct-devel
 %defattr(-,root,root)
@@ -237,9 +239,11 @@ mv %buildroot/%_bindir/io_demo  %buildroot/%_libexecdir/%{name}/
 %_libdir/libuct.so
 %dir %_libdir/ucx/
 %_libdir/ucx/libuct_*.so
+%_libdir/ucx/libucx_perftest_mad.so
 %_libdir/pkgconfig/ucx-uct.pc
 %_libdir/pkgconfig/ucx-cma.pc
 %_libdir/pkgconfig/ucx-ib.pc
+%_libdir/pkgconfig/ucx-ib-mlx5.pc
 %_libdir/pkgconfig/ucx-rdmacm.pc
 
 %changelog