Skip to content
Snippets Groups Projects
Commit b3d23010 authored by Pierre Lespagnol's avatar Pierre Lespagnol Committed by Sébastien Blin
Browse files

media_encoder: hardware frame refactor 

Fix unlinked hardware encoder/decoder cases (Conference, plugin, ...)

Change-Id: I045de3d24d89e93141309b6b353cd0bcabd8509f
parent ce56e13e
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
src/media/media_encoder.cpp
+ 105
− 66
View file @ b3d23010
......@@ -305,7 +305,8 @@ MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo, AVBufferRef* fr
throw MediaEncoderException("buffer too small");
scaledFrameBuffer_.reserve(scaledFrameBufferSize_);
scaledFrame_.setFromMemory(scaledFrameBuffer_.data(), format, width, height);
scaledFrame_ = std::make_shared<VideoFrame>();
scaledFrame_->setFromMemory(scaledFrameBuffer_.data(), format, width, height);
}
#endif // ENABLE_VIDEO
......@@ -357,89 +358,44 @@ MediaEncoder::startIO()
#ifdef ENABLE_VIDEO
int
MediaEncoder::encode(VideoFrame& input, bool is_keyframe, int64_t frame_number)
MediaEncoder::encode(const std::shared_ptr<VideoFrame>& input, bool is_keyframe, int64_t frame_number)
{
if (!initialized_) {
initStream(videoCodec_, input.pointer()->hw_frames_ctx);
initStream(videoCodec_, input->pointer()->hw_frames_ctx);
startIO();
}
AVFrame* frame;
std::shared_ptr<VideoFrame> output;
#ifdef RING_ACCEL
auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input.format()));
bool isHardware = desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL);
try {
#if defined(TARGET_OS_IOS) && TARGET_OS_IOS
if (accel_) {
auto pix = accel_->getSoftwareFormat();
if (input.format() != pix) {
std::unique_ptr<VideoFrame> framePtr;
framePtr = scaler_.convertFormat(input, pix);
frame = framePtr->pointer();
} else {
frame = input.pointer();
}
} else {
#elif !defined(__APPLE__)
std::unique_ptr<VideoFrame> framePtr;
if (accel_ && accel_->isLinked() && isHardware) {
// Fully accelerated pipeline, skip main memory
// We have to check if the frame is hardware even if
// we are using linked HW encoder and decoder because after
// conference mixing the frame become software (prevent crashes)
frame = input.pointer();
} else if (isHardware) {
// Hardware decoded frame, transfer back to main memory
// Transfer to GPU if we have a hardware encoder
// Hardware decoders decode to NV12, but Jami's supported software encoders want YUV420P
AVPixelFormat pix = (accel_ ? accel_->getSoftwareFormat() : AV_PIX_FMT_NV12);
framePtr = video::HardwareAccel::transferToMainMemory(input, pix);
if (!accel_)
framePtr = scaler_.convertFormat(*framePtr, AV_PIX_FMT_YUV420P);
else
framePtr = accel_->transfer(*framePtr);
frame = framePtr->pointer();
} else if (accel_) {
// Software decoded frame with a hardware encoder, convert to accepted format first
auto pix = accel_->getSoftwareFormat();
if (input.format() != pix) {
framePtr = scaler_.convertFormat(input, pix);
framePtr = accel_->transfer(*framePtr);
} else {
framePtr = accel_->transfer(input);
}
frame = framePtr->pointer();
} else {
if (getHWFrame(input, output) < 0) {
JAMI_ERR("Fail to get hardware frame");
return -1;
}
#else
{
#endif // defined(TARGET_OS_IOS) && TARGET_OS_IOS
#endif
libav_utils::fillWithBlack(scaledFrame_.pointer());
scaler_.scale_with_aspect(input, scaledFrame_);
frame = scaledFrame_.pointer();
#ifdef RING_ACCEL
}
#endif
} catch (const std::runtime_error& e) {
JAMI_ERR("Accel failure: %s", e.what());
output = getScaledSWFrame(input);
#endif // RING_ACCEL
if (!output) {
JAMI_ERR("Fail to get frame");
return -1;
}
auto avframe = output->pointer();
AVCodecContext* enc = encoders_[currentStreamIdx_];
frame->pts = frame_number;
avframe->pts = frame_number;
if (enc->framerate.num != enc->time_base.den || enc->framerate.den != enc->time_base.num)
frame->pts /= (rational<int64_t>(enc->framerate) * rational<int64_t>(enc->time_base))
avframe->pts /= (rational<int64_t>(enc->framerate) * rational<int64_t>(enc->time_base))
.real<int64_t>();
if (is_keyframe) {
frame->pict_type = AV_PICTURE_TYPE_I;
frame->key_frame = 1;
avframe->pict_type = AV_PICTURE_TYPE_I;
avframe->key_frame = 1;
} else {
frame->pict_type = AV_PICTURE_TYPE_NONE;
frame->key_frame = 0;
avframe->pict_type = AV_PICTURE_TYPE_NONE;
avframe->key_frame = 0;
}
return encode(frame, currentStreamIdx_);
return encode(avframe, currentStreamIdx_);
}
#endif // ENABLE_VIDEO
......@@ -1168,4 +1124,87 @@ MediaEncoder::testH265Accel()
return "";
}
int
MediaEncoder::getHWFrame(const std::shared_ptr<VideoFrame>& input, std::shared_ptr<VideoFrame>& output)
{
try {
#if defined(TARGET_OS_IOS) && TARGET_OS_IOS
// iOS
if (accel_) {
auto pix = accel_->getSoftwareFormat();
if (input->format() != pix) {
output = scaler_.convertFormat(*input.get(), pix);
} else {
// Fully accelerated pipeline, skip main memory
output = input;
}
} else {
output = getScaledSWFrame(*input.get());
}
#elif !defined(__APPLE__)
// Other Platforms
auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input->format()));
bool isHardware = desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL);
if (accel_ && accel_->isLinked() && isHardware) {
// Fully accelerated pipeline, skip main memory
output = input;
} else if (isHardware) {
// Hardware decoded frame, transfer back to main memory
// Transfer to GPU if we have a hardware encoder
// Hardware decoders decode to NV12, but Jami's supported software encoders want YUV420P
output = getUnlinkedHWFrame(*input.get());
} else if (accel_) {
// Software decoded frame with a hardware encoder, convert to accepted format first
output = getHWFrameFromSWFrame(*input.get());
} else {
output = getScaledSWFrame(*input.get());
}
#else
// macOS
output = getScaledSWFrame(*input.get());
#endif
} catch (const std::runtime_error& e) {
JAMI_ERR("Accel failure: %s", e.what());
return -1;
}
return 0;
}
std::shared_ptr<VideoFrame>
MediaEncoder::getUnlinkedHWFrame(const VideoFrame& input)
{
AVPixelFormat pix = (accel_ ? accel_->getSoftwareFormat() : AV_PIX_FMT_NV12);
std::shared_ptr<VideoFrame> framePtr = video::HardwareAccel::transferToMainMemory(input, pix);
if (!accel_) {
framePtr = scaler_.convertFormat(*framePtr, AV_PIX_FMT_YUV420P);
}
else {
framePtr = accel_->transfer(*framePtr);
}
return framePtr;
}
std::shared_ptr<VideoFrame>
MediaEncoder::getHWFrameFromSWFrame(const VideoFrame& input)
{
std::shared_ptr<VideoFrame> framePtr;
auto pix = accel_->getSoftwareFormat();
if (input.format() != pix) {
framePtr = scaler_.convertFormat(input, pix);
framePtr = accel_->transfer(*framePtr);
} else {
framePtr = accel_->transfer(input);
}
return framePtr;
}
std::shared_ptr<VideoFrame>
MediaEncoder::getScaledSWFrame(const VideoFrame& input)
{
libav_utils::fillWithBlack(scaledFrame_->pointer());
scaler_.scale_with_aspect(input, *scaledFrame_);
return scaledFrame_;
}
} // namespace jami
src/media/media_encoder.h
+ 6
− 2
View file @ b3d23010
......@@ -81,7 +81,7 @@ public:
bool send(AVPacket& packet, int streamIdx = -1);
#ifdef ENABLE_VIDEO
int encode(VideoFrame& input, bool is_keyframe, int64_t frame_number);
int encode(const std::shared_ptr<VideoFrame>& input, bool is_keyframe, int64_t frame_number);
#endif // ENABLE_VIDEO
int encodeAudio(AudioFrame& frame);
......@@ -134,6 +134,10 @@ private:
void initH263(AVCodecContext* encoderCtx, uint64_t br);
bool isDynBitrateSupported(AVCodecID codecid);
void initAccel(AVCodecContext* encoderCtx, uint64_t br);
int getHWFrame(const std::shared_ptr<VideoFrame>& input, std::shared_ptr<VideoFrame>& output);
std::shared_ptr<VideoFrame> getUnlinkedHWFrame(const VideoFrame& input);
std::shared_ptr<VideoFrame> getHWFrameFromSWFrame(const VideoFrame& input);
std::shared_ptr<VideoFrame> getScaledSWFrame(const VideoFrame& input);
std::vector<AVCodecContext*> encoders_;
AVFormatContext* outputCtx_ = nullptr;
......@@ -150,7 +154,7 @@ private:
#ifdef ENABLE_VIDEO
video::VideoScaler scaler_;
VideoFrame scaledFrame_;
std::shared_ptr<VideoFrame> scaledFrame_;
#endif // ENABLE_VIDEO
std::vector<uint8_t> scaledFrameBuffer_;
......
src/media/video/video_sender.cpp
+ 4
− 4
View file @ b3d23010
......@@ -72,16 +72,16 @@ VideoSender::~VideoSender()
}
void
VideoSender::encodeAndSendVideo(VideoFrame& input_frame)
VideoSender::encodeAndSendVideo(const std::shared_ptr<VideoFrame>& input_frame)
{
int angle = input_frame.getOrientation();
int angle = input_frame->getOrientation();
if (rotation_ != angle) {
rotation_ = angle;
if (changeOrientationCallback_)
changeOrientationCallback_(rotation_);
}
if (auto packet = input_frame.packet()) {
if (auto packet = input_frame->packet()) {
#if __ANDROID__
if (forceKeyFrame_) {
emitSignal<DRing::VideoSignal::RequestKeyFrame>();
......@@ -109,7 +109,7 @@ void
VideoSender::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
const std::shared_ptr<MediaFrame>& frame_p)
{
encodeAndSendVideo(*std::static_pointer_cast<VideoFrame>(frame_p));
encodeAndSendVideo(std::dynamic_pointer_cast<VideoFrame>(frame_p));
}
void
......
src/media/video/video_sender.h
+ 1
− 1
View file @ b3d23010
......@@ -69,7 +69,7 @@ private:
NON_COPYABLE(VideoSender);
void encodeAndSendVideo(VideoFrame&);
void encodeAndSendVideo(const std::shared_ptr<VideoFrame>&);
// encoder MUST be deleted before muxContext
std::unique_ptr<MediaIOHandle> muxContext_ = nullptr;
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment