From 737ede405b11a37fdd61d19cf25df296a0cb0b75
From: Cosmin Stejerean <cosmin@cosmin.at>
Date: Wed Dec 6 18:39:32 2023 +0800
Subject: avfilter/bwdif: account for chroma sub-sampling in min size calculation
References: https://bugzilla.opensuse.org/1223235
References: CVE-2023-49502

The current logic for detecting frames that are too small for the
algorithm does not account for chroma sub-sampling, and so a sample
where the luma plane is large enough, but the chroma planes are not
will not be rejected. In that event, a heap overflow will occur.

This change adjusts the logic to consider the chroma planes and makes
the change to all three bwdif implementations.

Fixes #10688

Signed-off-by: Cosmin Stejerean <cosmin@cosmin.at>
Reviewed-by: Thomas Mundt <tmundt75@gmail.com>
Signed-off-by: Philip Langdale <philipl@overt.org>

diff -Nura ffmpeg-5.1.4/libavfilter/vf_bwdif.c ffmpeg-5.1.4_new/libavfilter/vf_bwdif.c
--- ffmpeg-5.1.4/libavfilter/vf_bwdif.c	2023-11-10 07:38:51.000000000 +0800
+++ ffmpeg-5.1.4_new/libavfilter/vf_bwdif.c	2024-04-26 01:26:15.539021242 +0800
@@ -333,13 +333,14 @@
     if(yadif->mode&1)
         link->frame_rate = av_mul_q(link->src->inputs[0]->frame_rate, (AVRational){2,1});
 
-    if (link->w < 3 || link->h < 4) {
-        av_log(ctx, AV_LOG_ERROR, "Video of less than 3 columns or 4 lines is not supported\n");
+    yadif->csp = av_pix_fmt_desc_get(link->format);
+    yadif->filter = filter;
+
+    if (AV_CEIL_RSHIFT(link->w, yadif->csp->log2_chroma_w) < 3 || AV_CEIL_RSHIFT(link->h, yadif->csp->log2_chroma_h) < 4) {
+        av_log(ctx, AV_LOG_ERROR, "Video with planes less than 3 columns or 4 lines is not supported\n");
         return AVERROR(EINVAL);
     }
 
-    yadif->csp = av_pix_fmt_desc_get(link->format);
-    yadif->filter = filter;
     if (yadif->csp->comp[0].depth > 8) {
         s->filter_intra = filter_intra_16bit;
         s->filter_line  = filter_line_c_16bit;