ffmpeg-6/0001-avfilter-bwdif-account-for-chroma-sub-sampling-in-mi.patch

From 737ede405b11a37fdd61d19cf25df296a0cb0b75 Mon Sep 17 00:00:00 2001
From: Cosmin Stejerean <cosmin@cosmin.at>
Date: Wed, 6 Dec 2023 18:39:32 +0800
Subject: [PATCH] avfilter/bwdif: account for chroma sub-sampling in min size
 calculation

The current logic for detecting frames that are too small for the
algorithm does not account for chroma sub-sampling, and so a sample
where the luma plane is large enough, but the chroma planes are not
will not be rejected. In that event, a heap overflow will occur.

This change adjusts the logic to consider the chroma planes and makes
the change to all three bwdif implementations.

Fixes #10688

Signed-off-by: Cosmin Stejerean <cosmin@cosmin.at>
Reviewed-by: Thomas Mundt <tmundt75@gmail.com>
Signed-off-by: Philip Langdale <philipl@overt.org>
---
 libavfilter/vf_bwdif.c        |  9 +++++----
 libavfilter/vf_bwdif_cuda.c   | 11 ++++++-----
 libavfilter/vf_bwdif_vulkan.c | 11 +++++------
 3 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/libavfilter/vf_bwdif.c b/libavfilter/vf_bwdif.c
index 137cd5ef13..353cd0b61a 100644
--- a/libavfilter/vf_bwdif.c
+++ b/libavfilter/vf_bwdif.c
@@ -191,13 +191,14 @@ static int config_props(AVFilterLink *link)
         return ret;
     }
 
-    if (link->w < 3 || link->h < 4) {
-        av_log(ctx, AV_LOG_ERROR, "Video of less than 3 columns or 4 lines is not supported\n");
+    yadif->csp = av_pix_fmt_desc_get(link->format);
+    yadif->filter = filter;
+
+    if (AV_CEIL_RSHIFT(link->w, yadif->csp->log2_chroma_w) < 3 || AV_CEIL_RSHIFT(link->h, yadif->csp->log2_chroma_h) < 4) {
+        av_log(ctx, AV_LOG_ERROR, "Video with planes less than 3 columns or 4 lines is not supported\n");
         return AVERROR(EINVAL);
     }
 
-    yadif->csp = av_pix_fmt_desc_get(link->format);
-    yadif->filter = filter;
     ff_bwdif_init_filter_line(&s->dsp, yadif->csp->comp[0].depth);
 
     return 0;
diff --git a/libavfilter/vf_bwdif_cuda.c b/libavfilter/vf_bwdif_cuda.c
index a5ecfbadb6..418f15f989 100644
--- a/libavfilter/vf_bwdif_cuda.c
+++ b/libavfilter/vf_bwdif_cuda.c
@@ -296,15 +296,16 @@ static int config_output(AVFilterLink *link)
         link->frame_rate = av_mul_q(ctx->inputs[0]->frame_rate,
                                     (AVRational){2, 1});
 
-    if (link->w < 3 || link->h < 3) {
-        av_log(ctx, AV_LOG_ERROR, "Video of less than 3 columns or lines is not supported\n");
-        ret = AVERROR(EINVAL);
-        goto exit;
-    }
 
     y->csp = av_pix_fmt_desc_get(output_frames->sw_format);
     y->filter = filter;
 
+    if (AV_CEIL_RSHIFT(link->w, y->csp->log2_chroma_w) < 3 || AV_CEIL_RSHIFT(link->h, y->csp->log2_chroma_h) < 3) {
+        av_log(ctx, AV_LOG_ERROR, "Video with planes less than 3 columns or lines is not supported\n");
+        ret = AVERROR(EINVAL);
+        goto exit;
+    }
+
     ret = CHECK_CU(cu->cuCtxPushCurrent(s->hwctx->cuda_ctx));
     if (ret < 0)
         goto exit;
diff --git a/libavfilter/vf_bwdif_vulkan.c b/libavfilter/vf_bwdif_vulkan.c
index 690a89c4ba..c51df9aa26 100644
--- a/libavfilter/vf_bwdif_vulkan.c
+++ b/libavfilter/vf_bwdif_vulkan.c
@@ -362,15 +362,14 @@ static int bwdif_vulkan_config_output(AVFilterLink *outlink)
         outlink->frame_rate = av_mul_q(avctx->inputs[0]->frame_rate,
                                        (AVRational){2, 1});
 
-    if (outlink->w < 4 || outlink->h < 4) {
-        av_log(avctx, AV_LOG_ERROR, "Video of less than 4 columns or lines is not "
-               "supported\n");
-        return AVERROR(EINVAL);
-    }
-
     y->csp = av_pix_fmt_desc_get(vkctx->frames->sw_format);
     y->filter = bwdif_vulkan_filter_frame;
 
+    if (AV_CEIL_RSHIFT(outlink->w, y->csp->log2_chroma_w) < 4 || AV_CEIL_RSHIFT(outlink->h, y->csp->log2_chroma_h) < 4) {
+        av_log(avctx, AV_LOG_ERROR, "Video with planes less than 4 columns or lines is not supported\n");
+        return AVERROR(EINVAL);
+    }
+
     return init_filter(avctx);
 }
 
-- 
2.44.0