kpipewire6/0002-Add-encoder-using-libopenh264.patch

From 43ab595c28e031f38bc92bea4cf475de64021958 Mon Sep 17 00:00:00 2001
From: Fabian Vogt <fabian@ritter-vogt.de>
Date: Sat, 6 Jul 2024 16:40:42 +0200
Subject: [PATCH 2/2] Add encoder using libopenh264

On some distributions, libopenh264 is the only encoder available OOTB.
Add support for it and use it as fallback.

BUG: 476187
(cherry picked from commit e17793a3b023f26411001093bb2d5934adf715c7)
---
 src/CMakeLists.txt                |   1 +
 src/libopenh264encoder.cpp        | 106 ++++++++++++++++++++++++++++++
 src/libopenh264encoder_p.h        |  28 ++++++++
 src/pipewirebaseencodedstream.cpp |   2 +-
 src/pipewireproduce.cpp           |  11 ++++
 5 files changed, 147 insertions(+), 1 deletion(-)
 create mode 100644 src/libopenh264encoder.cpp
 create mode 100644 src/libopenh264encoder_p.h

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index e96f52b..3126528 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -135,6 +135,7 @@ add_library(KPipeWireRecord ${kpipewirerecord_SRCS}
                             encoder.cpp
                             h264vaapiencoder.cpp
                             libx264encoder.cpp
+                            libopenh264encoder.cpp
                             libvpxencoder.cpp
                             libvpxvp9encoder.cpp
 )
diff --git a/src/libopenh264encoder.cpp b/src/libopenh264encoder.cpp
new file mode 100644
index 0000000..6d4c6a1
--- /dev/null
+++ b/src/libopenh264encoder.cpp
@@ -0,0 +1,106 @@
+/*
+    SPDX-FileCopyrightText: 2023 Aleix Pol Gonzalez <aleixpol@kde.org>
+    SPDX-FileCopyrightText: 2023 Marco Martin <mart@kde.org>
+    SPDX-FileCopyrightText: 2023 Arjen Hiemstra <ahiemstra@heimr.nl>
+    SPDX-FileCopyrightText: 2024 Fabian Vogt <fabian@ritter-vogt.de>
+
+    SPDX-License-Identifier: LGPL-2.1-only OR LGPL-3.0-only OR LicenseRef-KDE-Accepted-LGPL
+*/
+
+#include "libopenh264encoder_p.h"
+
+#include <QSize>
+#include <QThread>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavfilter/buffersink.h>
+#include <libavfilter/buffersrc.h>
+#include <libavutil/pixfmt.h>
+}
+
+#include "logging_record.h"
+
+LibOpenH264Encoder::LibOpenH264Encoder(H264Profile profile, PipeWireProduce *produce)
+    : SoftwareEncoder(produce)
+    , m_profile(profile)
+{
+}
+
+bool LibOpenH264Encoder::initialize(const QSize &size)
+{
+    createFilterGraph(size);
+
+    auto codec = avcodec_find_encoder_by_name("libopenh264");
+    if (!codec) {
+        qCWarning(PIPEWIRERECORD_LOGGING) << "libopenh264 codec not found";
+        return false;
+    }
+
+    m_avCodecContext = avcodec_alloc_context3(codec);
+    if (!m_avCodecContext) {
+        qCWarning(PIPEWIRERECORD_LOGGING) << "Could not allocate video codec context";
+        return false;
+    }
+
+    Q_ASSERT(!size.isEmpty());
+    m_avCodecContext->width = size.width();
+    m_avCodecContext->height = size.height();
+    m_avCodecContext->max_b_frames = 0;
+    m_avCodecContext->gop_size = 100;
+    m_avCodecContext->pix_fmt = AV_PIX_FMT_YUV420P;
+    m_avCodecContext->time_base = AVRational{1, 1000};
+
+    if (m_quality) {
+        // "q" here stands for "quantization", but that effectively impacts quality.
+        m_avCodecContext->qmin = m_avCodecContext->qmax = percentageToAbsoluteQuality(m_quality);
+    }
+
+    switch (m_profile) {
+    case H264Profile::Baseline:
+        // libopenh264 only does constrained baseline.
+        // There's a bug in the ffmpeg -> openh264 interface though:
+        // ffmpeg expects CONSTRAINED_BASELINE from the application and
+        // passes that through, but libopenh264 only allows BASELINE.
+        // Until that bug is fixed there'll always be a warning that the
+        // profile is not supported (https://github.com/cisco/openh264/issues/3613)
+        m_avCodecContext->profile = FF_PROFILE_H264_CONSTRAINED_BASELINE;
+        break;
+    case H264Profile::Main:
+        m_avCodecContext->profile = FF_PROFILE_H264_MAIN;
+        break;
+    case H264Profile::High:
+        m_avCodecContext->profile = FF_PROFILE_H264_HIGH;
+        break;
+    }
+
+    AVDictionary *options = nullptr;
+    av_dict_set_int(&options, "threads", qMin(16, QThread::idealThreadCount()), 0);
+    applyEncodingPreference(options);
+
+    if (int result = avcodec_open2(m_avCodecContext, codec, &options); result < 0) {
+        qCWarning(PIPEWIRERECORD_LOGGING) << "Could not open codec" << av_err2str(result);
+        return false;
+    }
+
+    return true;
+}
+
+int LibOpenH264Encoder::percentageToAbsoluteQuality(const std::optional<quint8> &quality)
+{
+    if (!quality) {
+        return -1;
+    }
+
+    // 1-51 (incl.), lower is better
+    return 51 - (m_quality.value() / 100.0) * 50;
+}
+
+void LibOpenH264Encoder::applyEncodingPreference(AVDictionary *options)
+{
+    SoftwareEncoder::applyEncodingPreference(options);
+    // Disable motion estimation, not great while dragging windows but speeds up encoding by an order of magnitude
+    av_dict_set(&options, "flags", "+mv4", 0);
+    // Disable in-loop filtering
+    av_dict_set_int(&options, "loopfilter", 0, 0);
+}
diff --git a/src/libopenh264encoder_p.h b/src/libopenh264encoder_p.h
new file mode 100644
index 0000000..fdacf14
--- /dev/null
+++ b/src/libopenh264encoder_p.h
@@ -0,0 +1,28 @@
+/*
+    SPDX-FileCopyrightText: 2023 Aleix Pol Gonzalez <aleixpol@kde.org>
+    SPDX-FileCopyrightText: 2023 Marco Martin <mart@kde.org>
+    SPDX-FileCopyrightText: 2023 Arjen Hiemstra <ahiemstra@heimr.nl>
+    SPDX-FileCopyrightText: 2024 Fabian Vogt <fabian@ritter-vogt.de>
+
+    SPDX-License-Identifier: LGPL-2.1-only OR LGPL-3.0-only OR LicenseRef-KDE-Accepted-LGPL
+*/
+
+#include "encoder_p.h"
+
+/**
+ * A software encoder that uses ffmpeg + libopenh264 to encode to H.264.
+ */
+class LibOpenH264Encoder : public SoftwareEncoder
+{
+public:
+    LibOpenH264Encoder(H264Profile profile, PipeWireProduce *produce);
+
+    bool initialize(const QSize &size) override;
+
+protected:
+    int percentageToAbsoluteQuality(const std::optional<quint8> &quality) override;
+    void applyEncodingPreference(AVDictionary *options) override;
+
+private:
+    H264Profile m_profile = H264Profile::Main;
+};
diff --git a/src/pipewirebaseencodedstream.cpp b/src/pipewirebaseencodedstream.cpp
index 553c334..814d8d9 100644
--- a/src/pipewirebaseencodedstream.cpp
+++ b/src/pipewirebaseencodedstream.cpp
@@ -225,7 +225,7 @@ QList<PipeWireBaseEncodedStream::Encoder> PipeWireBaseEncodedStream::suggestedEn
                 && avcodec_find_encoder_by_name("h264_vaapi")) {
                 return false;
             } else {
-                return !avcodec_find_encoder_by_name("libx264");
+                return !(avcodec_find_encoder_by_name("libx264") || avcodec_find_encoder_by_name("libopenh264"));
             }
         default:
             return true;
diff --git a/src/pipewireproduce.cpp b/src/pipewireproduce.cpp
index 416bcd3..52594e6 100644
--- a/src/pipewireproduce.cpp
+++ b/src/pipewireproduce.cpp
@@ -16,6 +16,7 @@
 #include <qstringliteral.h>
 
 #include "h264vaapiencoder_p.h"
+#include "libopenh264encoder_p.h"
 #include "libvpxencoder_p.h"
 #include "libvpxvp9encoder_p.h"
 #include "libx264encoder_p.h"
@@ -295,6 +296,16 @@ std::unique_ptr<Encoder> PipeWireProduce::makeEncoder()
                 return softwareEncoder;
             }
         }
+
+        // Try libopenh264 last, it's slower and has less features.
+        if (forcedEncoder.isNull() || forcedEncoder == u"libopenh264") {
+            auto softwareEncoder = std::make_unique<LibOpenH264Encoder>(profile, this);
+            softwareEncoder->setQuality(m_quality);
+            softwareEncoder->setEncodingPreference(m_encodingPreference);
+            if (softwareEncoder->initialize(size)) {
+                return softwareEncoder;
+            }
+        }
         break;
     }
     case PipeWireBaseEncodedStream::VP8: {
-- 
2.45.2