Sync from SUSE:SLFO:Main jdupes revision bb00d6ff97e806ab31527afd6bf010e9

.gitattributes

@@ -0,0 +1,23 @@
+## Default LFS
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.bsp filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.gem filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.jar filter=lfs diff=lfs merge=lfs -text
+*.lz filter=lfs diff=lfs merge=lfs -text
+*.lzma filter=lfs diff=lfs merge=lfs -text
+*.obscpio filter=lfs diff=lfs merge=lfs -text
+*.oxt filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.rpm filter=lfs diff=lfs merge=lfs -text
+*.tbz filter=lfs diff=lfs merge=lfs -text
+*.tbz2 filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.ttf filter=lfs diff=lfs merge=lfs -text
+*.txz filter=lfs diff=lfs merge=lfs -text
+*.whl filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text

jdupes.changes

@@ -0,0 +1,153 @@
+-------------------------------------------------------------------
+Thu Feb  9 17:13:54 UTC 2023 - Dirk Müller <dmueller@suse.com>
+
+- update to 1.21.3:
+  * Major performance fix in the double traversal prevention tree
+    code
+  * Added undocumented '-9' benchmark option for testing traversal
+    code
+  * Extra error checks to try to finally solve a very rare crash
+    bug
+  * Tiny code size reduction by discarding unused xxHash code
+  * jody_hash re-added as a build-time option
+  * Many options/features/safety checks can now be selectively
+    compiled out
+  * New 'make BARE_BONES=1' option builds the most minimal jdupes
+    possible
+  * Fix exit behavior when no valid directories are given
+  * Only act on "normal" files and directories 
+  
+-------------------------------------------------------------------
+Tue Dec  6 21:35:51 UTC 2022 - Dirk Müller <dmueller@suse.com>
+
+- update to 1.21.1:
+  * Reinstate '-I/--isolate' by popular demand; use at your own risk!
+  * Expect to lose data if you use this feature. It is strongly susceptible
+    to the documented "triangle problem" and absolutely does not protect
+    files in each directory from matching with one another. Don't get mad at
+    me if you use it and see a cloud of smoke come out of your disk array.
+  * Remove '-I/--isolate' which has never worked correctly
+  * Fix compiling when NO_HARDLINKS and NO_SYMLINKS are both defined
+  * Increased stack size limits to enable deeper recursion without crashing
+  * Fixes to make compilation under Cygwin (instead of MSYS2 MinGW) work
+  * Remove the temporary '-X help' warning about changes in functionality
+  * Some minor under-the-hood changes for future enhancements
+
+-------------------------------------------------------------------
+Mon Mar 14 22:06:26 UTC 2022 - Dirk Müller <dmueller@suse.com>
+
+- add jdupes_wrapper to speedup symlinks
+- add new %suse_symlink_dupes and %suse_hardlink_dupes macros
+  which are more descriptive than a %jdupes or %fdupes 
+
+-------------------------------------------------------------------
+Wed Mar  9 21:54:15 UTC 2022 - Dirk Müller <dmueller@suse.com>
+
+- update to 1.20.2:
+  * Interactive deletion now offers "link set" options too
+  * Interactive deletion now assumes 'a' when you just hit [ENTER]
+- add jdupes rpm macro for dropin %%fdupes replacement
+- run tests
+
+-------------------------------------------------------------------
+Thu Jul 08 20:24:03 UTC 2021 - idesmi@protonmail.com
+
+- Update to version 1.20.0:
+  * v1.20.0 release
+  * Update copyright years and financial support information
+  * Update README.md and manpage: long options and copyright years
+  * jody_*: update/add copyright information
+  * oom.c/h: change license to Creative Commons 0 (public domain)
+  * stupid_dupes.sh: update version information and copyright years
+  * Use consistent hyphenation in long options; remove deprecated -n
+  * Remove redundant warning about another option (#172)
+  * Fix -P partial printout when progress indicator is enabled
+  * INSTALL: More stern warnings to not use LOW_MEMORY inappropriately
+
+-------------------------------------------------------------------
+Sun Nov  1 23:22:39 UTC 2020 - malcolmlewis@opensuse.org
+
+- Updated to version 1.19.0:
+  * Make -X size[+-=] an inclusion filter instead (CHECK YOUR SHELL
+    SCRIPTS).
+  * Fix -X noext/only extension matching bugs.
+  * Remove deprecated -x/--xsize option and some undocumented
+    redundant options.
+  * Name sorting is now case-insensitive.
+  * Disable -K/--skiphash which was accidentally left active but
+    doesn't work.
+  * When sorting by modify time and there's a tie, break it using
+    name sorting.
+  * jdupes-standalone has been removed due to falling too far out
+    of sync.
+  * Embedded build date option has been removed.
+- Changes from version 1.18.2:
+  * Add -U/--notravcheck to skip double-traversal safety (for
+    Google Drive FS).
+- Changes from version 1.18.0
+  * Add -X newer/older extfilters to reject files by modification
+    date.
+- Changes from version 1.17.0:
+  * Rewrite dedupe code from scratch, probably fixing all dedupe
+    bugs.
+  * extfilter: add substring match filter for file paths.
+  * Add -u/--printunique option to print all non-duplicates
+    (unmatched files).
+  * Dedupe-blacklisted kernel version check now happens before
+    work is done.
+- Changes from version 1.16.0:
+  * Add -X noext/onlyext filters to exclude/require specific file
+    extension(s).
+  * Added in-depth help text for -X/--extfilter (use -X help to
+    view).
+  * This version still has BTRFS dedupe issues and file add-by-name
+    disabled.
+- Changes from version 1.15.0:
+  * Disable single file addition on the command line for safety.
+- CHanges from version 1.14.1:
+  * Fix some compilation issues.
+  * Add example shell scripts for processing piped jdupes output.
+  * Fix some swapped/mangled help text in program and
+    documentation.
+  *LOW_MEMORY compiles exclude more stuff to further reduce usage.
+
+-------------------------------------------------------------------
+Thu Mar 19 03:27:55 UTC 2020 - malcolmlewis@opensuse.org
+
+- Updated to version 1.14.0
+  * Long option --exclude is deprecated and renamed --extfilter.
+  * BTRFS compile options have been generalized to dedupe,
+   (see README).
+  * Fix a bug in 1.13.3 where many options caused an inappropriate
+    exit.
+- Changes from version 1.13.3:
+  * Fix the behavior of the -I/--isolate option.
+  * Move BTRFS-specific dedupe interfaces to general Linux 4.5+
+    interfaces.
+  * Change BTRFS compilation flag name (see README).
+  * Fix FS dedupe only working on the first 16 MiB of files.
+  * Add FS dedupe static header for when this header is missing.
+  * Correct several copyright dates and comments.
+
+-------------------------------------------------------------------
+Tue Aug  6 20:02:38 UTC 2019 - malcolmlewis@opensuse.org
+
+- Updated to version 1.13.2:
+  * Fix Unicode and escapes in JSON output,
+    (gh#jbruchon/jdupes#109).
+
+------------------------------------------------------------------
+Sun Jul 14 23:44:59 UTC 2019 - malcolmlewis@opensuse.org
+
+- Updated to version 1.13.1:
+  * Fix an incorrect NULL pointer check.
+- Changes from version 1.13:
+  * Add new option -j/--json for JSON (machine-readable) output.
+  * /usr/local is now the default PREFIX in Makefile.
+  * Minor performance optimizations.
+  * A few minor bug fixes.
+
+-------------------------------------------------------------------
+Sun Apr 28 04:35:40 UTC 2019 - malcolmlewis@opensuse.org
+
+- Initial build.

jdupes.spec

@@ -0,0 +1,65 @@
+#
+# spec file for package jdupes
+#
+# Copyright (c) 2023 SUSE LLC
+# Copyright (c) 2019-2020 Malcolm J Lewis <malcolmlewis@opensuse.org>
+#
+# All modifications and additions to the file contributed by third parties
+# remain the property of their copyright owners, unless otherwise agreed
+# upon. The license for this file, and modifications and additions to the
+# file, is the same license as for the pristine package itself (unless the
+# license for the pristine package is not an Open Source License, in which
+# case the license is the MIT License). An "Open Source License" is a
+# license that conforms to the Open Source Definition (Version 1.9)
+# published by the Open Source Initiative.
+
+# Please submit bugfixes or comments via https://bugs.opensuse.org/
+#
+
+
+Name:           jdupes
+Version:        1.21.3
+Release:        0
+Summary:        A powerful duplicate file finder and an enhanced fork of 'fdupes'
+License:        MIT
+Group:          Productivity/File utilities
+URL:            https://github.com/jbruchon/jdupes
+Source0:        https://github.com/jbruchon/jdupes/archive/refs/tags/v%{version}.tar.gz
+Source1:        macros.jdupes
+Source2:        jdupes_wrapper.cpp
+BuildRequires:  gcc-c++
+
+%description
+A program for identifying and taking actions upon duplicate files.
+
+A WORD OF WARNING: jdupes IS NOT a drop-in compatible replacement for fdupes!
+Do not blindly replace fdupes with jdupes in scripts and expect everything to
+work the same way. Option availability and meanings differ between the two
+programs.
+
+%prep
+%setup -q
+
+%build
+make %{?_smp_mflags} \
+     ENABLE_DEDUPE=1 \
+     STATIC_DEDUPE_H=1
+g++ %{optflags} -O2 -Wall %{SOURCE2} -o jdupes_wrapper
+
+%install
+make DESTDIR=%{buildroot} PREFIX=%{_prefix} install
+install -D -m644 %{SOURCE1} %{buildroot}%{_rpmmacrodir}/macros.%{name}
+install -D -m755 jdupes_wrapper  %{buildroot}/usr/lib/rpm/jdupes_wrapper
+
+%check
+./jdupes -q -r testdir
+
+%files
+%license LICENSE
+%doc CHANGES README.md
+%{_bindir}/%{name}
+%{_mandir}/man1/%{name}.1%{?ext_man}
+%{_rpmmacrodir}/macros.%{name}
+/usr/lib/rpm/jdupes_wrapper
+
+%changelog

jdupes_wrapper.cpp

@@ -0,0 +1,167 @@
+/*
+ * A little helper to wrap around jdupes and create hard/soft links of the
+ * dups found. Used in openSUSE rpm.
+ *
+ * Copyright 2022 Jiri Slaby <jslaby@suse.cz>
+ *           2022 Stephan Kulow <coolo@suse.de>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <algorithm>
+#include <array>
+#include <iostream>
+#include <list>
+#include <map>
+#include <string>
+#include <sys/param.h>
+#include <sys/stat.h>
+#include <tuple>
+#include <unistd.h>
+#include <utility>
+#include <vector>
+
+typedef std::map<ino_t, std::vector<std::string>> dups_map;
+typedef std::pair<ino_t, size_t> nlink_pair;
+
+bool cmp_nlink(const nlink_pair& a, const nlink_pair& b)
+{
+    return a.second > b.second;
+}
+
+void sort_by_count(const dups_map& in, std::vector<ino_t>& out)
+{
+    out.clear();
+    std::list<nlink_pair> nlinks;
+    for (auto it = in.cbegin(); it != in.cend(); ++it) {
+        nlinks.push_back(std::make_pair(it->first, it->second.size()));
+    }
+    nlinks.sort(cmp_nlink);
+    for (auto it = nlinks.cbegin(); it != nlinks.cend(); ++it) {
+        out.push_back(it->first);
+    }
+}
+
+void link_file(const std::string& file, const std::string& target, bool symlink)
+{
+    std::cout << "Linking " << file << " -> " << target << std::endl;
+    if (unlink(file.c_str())) {
+        std::cerr << "Removing '" << file << "' failed." << std::endl;
+        exit(1);
+    }
+    int ret;
+    if (symlink) {
+        ret = ::symlink(target.c_str(), file.c_str());
+    } else {
+        ret = link(target.c_str(), file.c_str());
+    }
+    if (ret) {
+        std::cerr << "Linking '" << file << "' failed." << std::endl;
+        exit(1);
+    }
+}
+
+void handle_dups(const dups_map& dups, const std::string& buildroot, bool symlink)
+{
+    // all are hardlinks to the same data
+    if (dups.size() < 2)
+        return;
+    std::vector<ino_t> sorted;
+    sort_by_count(dups, sorted);
+    auto inodes = sorted.begin();
+    std::string target = dups.at(*inodes).front();
+    if (symlink) {
+        target.replace(0, buildroot.length(), "");
+    }
+
+    for (++inodes; inodes != sorted.end(); ++inodes) {
+        const std::vector<std::string> files = dups.at(*inodes);
+        for (auto it = files.begin(); it != files.end(); ++it) {
+            link_file(*it, target, symlink);
+        }
+    }
+}
+
+int main(int argc, char** argv)
+{
+    bool symlink = false;
+    std::string root;
+    std::string buildroot;
+    while (1) {
+        int result = getopt(argc, argv, "sb:");
+        if (result == -1)
+            break; /* end of list */
+        switch (result) {
+        case 's':
+            symlink = true;
+            break;
+        case 'b':
+            buildroot = optarg;
+            break;
+        default: /* unknown */
+            break;
+        }
+    }
+    if (buildroot.empty()) {
+        if (symlink) {
+            std::cerr << "Missing -b argument to remove bootroot from symlink targets";
+            return 1;
+        }
+        // eliminate final slash from directory argument
+        if (buildroot.back() == '/') {
+            buildroot.pop_back();
+        }
+    }
+    if (optind < argc) {
+        root = argv[optind++];
+    } else {
+        std::cerr << "Missing directory argument.";
+    }
+    if (optind < argc) {
+        std::cerr << "Too many arguments.";
+        return 1;
+    }
+    /* jdupes options used:
+       -q: hide progress indicator
+       -p: don't consider files with different owner/group or permission bits as duplicates
+       -o name: output order of duplicates
+       -r: follow subdirectories
+       -H: also report hard links as duplicates
+    */
+    std::string command = "jdupes -q -p -o name";
+    if (!symlink) {
+        /* if we create symlinks, avoid looking at hard links being duplicated. This way
+           jdupes is faster and won't break them up anyway */
+        command += " -H";
+    }
+    command += " -r '" + root + "'";
+    FILE* pipe = popen(command.c_str(), "r");
+    if (!pipe) {
+        throw std::runtime_error("popen() failed!");
+    }
+    std::array<char, MAXPATHLEN> buffer;
+    dups_map dups;
+    while (fgets(buffer.data(), buffer.size(), pipe) != nullptr) {
+        std::string line = buffer.data();
+        if (line.length() < 2) {
+            handle_dups(dups, buildroot, symlink);
+            dups.clear();
+            continue;
+        }
+        if (line.back() != '\n') {
+            std::cerr << "Too long lines? '" << line << "'" << std::endl;
+            return 1;
+        }
+        line.pop_back();
+
+        struct stat sb;
+        if (stat(line.c_str(), &sb)) {
+            std::cerr << "Stat on '" << buffer.data() << "' failed" << std::endl;
+            return 1;
+        }
+        dups[sb.st_ino].push_back(line);
+    }
+    pclose(pipe);
+
+    return 0;
+}

macros.jdupes

@@ -0,0 +1,4 @@
+%suse_symlink_dupes /usr/lib/rpm/jdupes_wrapper -b %{buildroot} -s
+%suse_hardlink_dupes jdupes -q -p -H -o name -L -r %1
+
+%fdupes /usr/lib/rpm/jdupes_wrapper -b %{buildroot}