Commit c499a22b authored by Jinbo Li's avatar Jinbo Li Committed by Philippe Gorley

encoder: add error checking to videoFrameSize

av_image_get_buffer_size can return a negative error code. Prevents
casting such an error to a large positive integer that may cause a
bad_alloc exception on the subsequent buffer allocation.

Change-Id: Ie8a7987dcf16957a21496e791c34957bb2e35bfc
parent 7ae0214d
......@@ -34,7 +34,7 @@ namespace ring {
//=== HELPERS ==================================================================
std::size_t
int
videoFrameSize(int format, int width, int height)
{
return av_image_get_buffer_size((AVPixelFormat)format, width, height, 1);
......
......@@ -42,7 +42,7 @@ using AudioFrame = DRing::AudioFrame;
using VideoFrame = DRing::VideoFrame;
// Some helpers
std::size_t videoFrameSize(int format, int width, int height);
int videoFrameSize(int format, int width, int height);
#endif // RING_VIDEO
......
......@@ -286,7 +286,9 @@ MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo, std::string para
const int height = encoderCtx->height;
const int format = encoderCtx->pix_fmt;
scaledFrameBufferSize_ = videoFrameSize(format, width, height);
if (scaledFrameBufferSize_ <= AV_INPUT_BUFFER_MIN_SIZE)
if (scaledFrameBufferSize_ < 0)
throw MediaEncoderException(("Could not compute buffer size: " + libav_utils::getError(scaledFrameBufferSize_)).c_str());
else if (scaledFrameBufferSize_ <= AV_INPUT_BUFFER_MIN_SIZE)
throw MediaEncoderException("buffer too small");
scaledFrameBuffer_.reserve(scaledFrameBufferSize_);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment