From 737ede405b11a37fdd61d19cf25df296a0cb0b75 Mon Sep 17 00:00:00 2001 From: Cosmin Stejerean Date: Wed, 6 Dec 2023 18:39:32 +0800 Subject: [PATCH] avfilter/bwdif: account for chroma sub-sampling in min size calculation References: https://bugzilla.opensuse.org/1223235 References: CVE-2023-49502 The current logic for detecting frames that are too small for the algorithm does not account for chroma sub-sampling, and so a sample where the luma plane is large enough, but the chroma planes are not will not be rejected. In that event, a heap overflow will occur. This change adjusts the logic to consider the chroma planes and makes the change to all three bwdif implementations. Fixes #10688 Signed-off-by: Cosmin Stejerean Reviewed-by: Thomas Mundt Signed-off-by: Philip Langdale --- libavfilter/vf_bwdif.c | 9 +++++---- libavfilter/vf_bwdif_cuda.c | 11 ++++++----- libavfilter/vf_bwdif_vulkan.c | 11 +++++------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/libavfilter/vf_bwdif.c b/libavfilter/vf_bwdif.c index 137cd5ef13..353cd0b61a 100644 --- a/libavfilter/vf_bwdif.c +++ b/libavfilter/vf_bwdif.c @@ -191,13 +191,14 @@ static int config_props(AVFilterLink *link) return ret; } - if (link->w < 3 || link->h < 4) { - av_log(ctx, AV_LOG_ERROR, "Video of less than 3 columns or 4 lines is not supported\n"); + yadif->csp = av_pix_fmt_desc_get(link->format); + yadif->filter = filter; + + if (AV_CEIL_RSHIFT(link->w, yadif->csp->log2_chroma_w) < 3 || AV_CEIL_RSHIFT(link->h, yadif->csp->log2_chroma_h) < 4) { + av_log(ctx, AV_LOG_ERROR, "Video with planes less than 3 columns or 4 lines is not supported\n"); return AVERROR(EINVAL); } - yadif->csp = av_pix_fmt_desc_get(link->format); - yadif->filter = filter; ff_bwdif_init_filter_line(&s->dsp, yadif->csp->comp[0].depth); return 0; diff --git a/libavfilter/vf_bwdif_cuda.c b/libavfilter/vf_bwdif_cuda.c index a5ecfbadb6..418f15f989 100644 --- a/libavfilter/vf_bwdif_cuda.c +++ b/libavfilter/vf_bwdif_cuda.c @@ -296,15 +296,16 @@ static int config_output(AVFilterLink *link) link->frame_rate = av_mul_q(ctx->inputs[0]->frame_rate, (AVRational){2, 1}); - if (link->w < 3 || link->h < 3) { - av_log(ctx, AV_LOG_ERROR, "Video of less than 3 columns or lines is not supported\n"); - ret = AVERROR(EINVAL); - goto exit; - } y->csp = av_pix_fmt_desc_get(output_frames->sw_format); y->filter = filter; + if (AV_CEIL_RSHIFT(link->w, y->csp->log2_chroma_w) < 3 || AV_CEIL_RSHIFT(link->h, y->csp->log2_chroma_h) < 3) { + av_log(ctx, AV_LOG_ERROR, "Video with planes less than 3 columns or lines is not supported\n"); + ret = AVERROR(EINVAL); + goto exit; + } + ret = CHECK_CU(cu->cuCtxPushCurrent(s->hwctx->cuda_ctx)); if (ret < 0) goto exit; diff --git a/libavfilter/vf_bwdif_vulkan.c b/libavfilter/vf_bwdif_vulkan.c index 690a89c4ba..c51df9aa26 100644 --- a/libavfilter/vf_bwdif_vulkan.c +++ b/libavfilter/vf_bwdif_vulkan.c @@ -362,15 +362,14 @@ static int bwdif_vulkan_config_output(AVFilterLink *outlink) outlink->frame_rate = av_mul_q(avctx->inputs[0]->frame_rate, (AVRational){2, 1}); - if (outlink->w < 4 || outlink->h < 4) { - av_log(avctx, AV_LOG_ERROR, "Video of less than 4 columns or lines is not " - "supported\n"); - return AVERROR(EINVAL); - } - y->csp = av_pix_fmt_desc_get(vkctx->frames->sw_format); y->filter = bwdif_vulkan_filter_frame; + if (AV_CEIL_RSHIFT(outlink->w, y->csp->log2_chroma_w) < 4 || AV_CEIL_RSHIFT(outlink->h, y->csp->log2_chroma_h) < 4) { + av_log(avctx, AV_LOG_ERROR, "Video with planes less than 4 columns or lines is not supported\n"); + return AVERROR(EINVAL); + } + return init_filter(avctx); } -- 2.44.0