From c499a22be44526cdcb8ff889a6158b08d5019cf1 Mon Sep 17 00:00:00 2001
From: jinboli <jinbo.li@savoirfairelinux.com>
Date: Thu, 22 Nov 2018 15:30:33 -0500
Subject: [PATCH] encoder: add error checking to videoFrameSize
av_image_get_buffer_size can return a negative error code. Prevents
casting such an error to a large positive integer that may cause a
bad_alloc exception on the subsequent buffer allocation.
Change-Id: Ie8a7987dcf16957a21496e791c34957bb2e35bfc
---
src/media/media_buffer.cpp | 2 +-
src/media/media_buffer.h | 2 +-
src/media/media_encoder.cpp | 4 +++-
3 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/src/media/media_buffer.cpp b/src/media/media_buffer.cpp
index cece1a7c4f..6acf2c2c61 100644
--- a/src/media/media_buffer.cpp
+++ b/src/media/media_buffer.cpp
@@ -34,7 +34,7 @@ namespace ring {
//=== HELPERS ==================================================================
-std::size_t
+int
videoFrameSize(int format, int width, int height)
{
return av_image_get_buffer_size((AVPixelFormat)format, width, height, 1);
diff --git a/src/media/media_buffer.h b/src/media/media_buffer.h
index 5bb8616f48..855467f76e 100644
--- a/src/media/media_buffer.h
+++ b/src/media/media_buffer.h
@@ -42,7 +42,7 @@ using AudioFrame = DRing::AudioFrame;
using VideoFrame = DRing::VideoFrame;
// Some helpers
-std::size_t videoFrameSize(int format, int width, int height);
+int videoFrameSize(int format, int width, int height);
#endif // RING_VIDEO
diff --git a/src/media/media_encoder.cpp b/src/media/media_encoder.cpp
index 7e59f3dd2b..80f97f5b07 100644
--- a/src/media/media_encoder.cpp
+++ b/src/media/media_encoder.cpp
@@ -286,7 +286,9 @@ MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo, std::string para
const int height = encoderCtx->height;
const int format = encoderCtx->pix_fmt;
scaledFrameBufferSize_ = videoFrameSize(format, width, height);
- if (scaledFrameBufferSize_ <= AV_INPUT_BUFFER_MIN_SIZE)
+ if (scaledFrameBufferSize_ < 0)
+ throw MediaEncoderException(("Could not compute buffer size: " + libav_utils::getError(scaledFrameBufferSize_)).c_str());
+ else if (scaledFrameBufferSize_ <= AV_INPUT_BUFFER_MIN_SIZE)
throw MediaEncoderException("buffer too small");
scaledFrameBuffer_.reserve(scaledFrameBufferSize_);
--
GitLab