encoder: add error checking to videoFrameSize

av_image_get_buffer_size can return a negative error code. Prevents
casting such an error to a large positive integer that may cause a
bad_alloc exception on the subsequent buffer allocation.

Change-Id: Ie8a7987dcf16957a21496e791c34957bb2e35bfc

src/media/media_buffer.cpp

@@ -34,7 +34,7 @@ namespace ring {
 
 //=== HELPERS ==================================================================
 
-std::size_t
+int
 videoFrameSize(int format, int width, int height)
 {
     return av_image_get_buffer_size((AVPixelFormat)format, width, height, 1);

src/media/media_buffer.h

@@ -42,7 +42,7 @@ using AudioFrame = DRing::AudioFrame;
 using VideoFrame = DRing::VideoFrame;
 
 // Some helpers
-std::size_t videoFrameSize(int format, int width, int height);
+int videoFrameSize(int format, int width, int height);
 
 #endif // RING_VIDEO
 

src/media/media_encoder.cpp

@@ -286,7 +286,9 @@ MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo, std::string para
         const int height = encoderCtx->height;
         const int format = encoderCtx->pix_fmt;
         scaledFrameBufferSize_ = videoFrameSize(format, width, height);
-        if (scaledFrameBufferSize_ <= AV_INPUT_BUFFER_MIN_SIZE)
+        if (scaledFrameBufferSize_ < 0)
+            throw MediaEncoderException(("Could not compute buffer size: " + libav_utils::getError(scaledFrameBufferSize_)).c_str());
+        else if (scaledFrameBufferSize_ <= AV_INPUT_BUFFER_MIN_SIZE)
             throw MediaEncoderException("buffer too small");
 
         scaledFrameBuffer_.reserve(scaledFrameBufferSize_);