Commit 52fbc755 authored by Philippe Gorley's avatar Philippe Gorley Committed by Adrien Béraud

accel: skip main memory when hardware reencoding

If the video input is being decoded on the hardware, use the hardware
frames directly when encoding. Skips the transfer back to software in
the video sender and the transfer to hardware in the encoder.

Falls back to using the main memory as a middle man if the link between
encoder and decoder fails.

Change-Id: I59850e95e6952df71a461aad8e7dddda65f05ffb
parent dfdee183
......@@ -227,9 +227,10 @@ MediaDecoder::setupStream(AVMediaType mediaType)
#ifdef RING_ACCEL
if (mediaType == AVMEDIA_TYPE_VIDEO) {
if (enableAccel_) {
accel_ = video::HardwareAccel::setupDecoder(decoderCtx_->codec_id);
accel_ = video::HardwareAccel::setupDecoder(decoderCtx_->codec_id,
decoderCtx_->width, decoderCtx_->height);
if (accel_) {
accel_->setDetails(decoderCtx_, &options_);
accel_->setDetails(decoderCtx_);
decoderCtx_->opaque = accel_.get();
}
} else if (Manager::instance().videoPreferences.getDecodingAccelerated()) {
......
......@@ -162,7 +162,7 @@ MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo)
{
if (systemCodecInfo.mediaType == MEDIA_AUDIO) {
audioCodec_ = systemCodecInfo.name;
return initStream(systemCodecInfo);
return initStream(systemCodecInfo, nullptr);
} else {
videoCodec_ = systemCodecInfo.name;
// TODO only support 1 audio stream and 1 video stream per encoder
......@@ -174,17 +174,17 @@ MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo)
}
int
MediaEncoder::initStream(const std::string& codecName)
MediaEncoder::initStream(const std::string& codecName, AVBufferRef* framesCtx)
{
const auto codecInfo = getSystemCodecContainer()->searchCodecByName(codecName, MEDIA_ALL);
if (codecInfo)
return initStream(*codecInfo);
return initStream(*codecInfo, framesCtx);
else
return -1;
}
int
MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo)
MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo, AVBufferRef* framesCtx)
{
AVCodec* outputCodec = nullptr;
AVCodecContext* encoderCtx = nullptr;
......@@ -192,7 +192,8 @@ MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo)
if (systemCodecInfo.mediaType == MEDIA_VIDEO) {
if (enableAccel_) {
if (accel_ = video::HardwareAccel::setupEncoder(
static_cast<AVCodecID>(systemCodecInfo.avcodecId), videoOpts_.width, videoOpts_.height)) {
static_cast<AVCodecID>(systemCodecInfo.avcodecId),
videoOpts_.width, videoOpts_.height, framesCtx)) {
outputCodec = avcodec_find_encoder_by_name(accel_->getCodecName().c_str());
}
} else {
......@@ -221,7 +222,7 @@ MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo)
#ifdef RING_ACCEL
if (accel_) {
accel_->setDetails(encoderCtx, &options_);
accel_->setDetails(encoderCtx);
encoderCtx->opaque = accel_.get();
}
#endif
......@@ -383,28 +384,49 @@ MediaEncoder::encode(VideoFrame& input, bool is_keyframe,
int64_t frame_number)
{
if (!initialized_) {
initStream(videoCodec_);
initStream(videoCodec_, input.pointer()->hw_frames_ctx);
startIO();
}
/* Prepare a frame suitable to our encoder frame format,
* keeping also the input aspect ratio.
*/
libav_utils::fillWithBlack(scaledFrame_.pointer());
scaler_.scale_with_aspect(input, scaledFrame_);
// Copy frame so the VideoScaler can still use the software frame (input)
VideoFrame copy;
copy.copyFrom(scaledFrame_);
AVFrame* frame;
#ifdef RING_ACCEL
auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input.format()));
bool isHardware = desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL);
std::unique_ptr<VideoFrame> framePtr;
if (accel_ && accel_->isLinked()) {
// Fully accelerated pipeline, skip main memory
frame = input.pointer();
} else if (isHardware) {
// Hardware decoded frame, transfer back to main memory
// Transfer to GPU if we have a hardware encoder
AVPixelFormat pix = (accel_ ? accel_->getSoftwareFormat() : AV_PIX_FMT_YUV420P);
framePtr = video::HardwareAccel::transferToMainMemory(input, pix);
if (accel_)
framePtr = accel_->transfer(*framePtr);
frame = framePtr->pointer();
} else if (accel_) {
// Software decoded frame with a hardware encoder, convert to accepted format first
auto pix = accel_->getSoftwareFormat();
if (input.format() != pix) {
framePtr = scaler_.convertFormat(input, pix);
framePtr = accel_->transfer(*framePtr);
} else {
framePtr = accel_->transfer(input);
}
frame = framePtr->pointer();
} else {
#endif
libav_utils::fillWithBlack(scaledFrame_.pointer());
scaler_.scale_with_aspect(input, scaledFrame_);
frame = scaledFrame_.pointer();
#ifdef RING_ACCEL
}
#endif
auto frame = copy.pointer();
AVCodecContext* enc = encoders_[currentStreamIdx_];
// ideally, time base is the inverse of framerate, but this may not always be the case
if (enc->framerate.num == enc->time_base.den && enc->framerate.den == enc->time_base.num)
frame->pts = frame_number;
else
frame->pts = (frame_number / (rational<int64_t>(enc->framerate) * rational<int64_t>(enc->time_base))).real<int64_t>();
frame->pts = frame_number;
if (enc->framerate.num != enc->time_base.den || enc->framerate.den != enc->time_base.num)
frame->pts /= (rational<int64_t>(enc->framerate) * rational<int64_t>(enc->time_base)).real<int64_t>();
if (is_keyframe) {
frame->pict_type = AV_PICTURE_TYPE_I;
......@@ -414,19 +436,6 @@ MediaEncoder::encode(VideoFrame& input, bool is_keyframe,
frame->key_frame = 0;
}
#ifdef RING_ACCEL
// NOTE needs to be at same scope as call to encode
std::unique_ptr<VideoFrame> framePtr;
if (accel_) {
framePtr = accel_->transfer(copy);
if (!framePtr) {
RING_ERR() << "Hardware encoding failure";
return -1;
}
frame = framePtr->pointer();
}
#endif
return encode(frame, currentStreamIdx_);
}
#endif // RING_VIDEO
......@@ -454,7 +463,7 @@ MediaEncoder::encode(AVFrame* frame, int streamIdx)
// Initialize on first video frame, or first audio frame if no video stream
bool isVideo = (frame->width > 0 && frame->height > 0);
if (isVideo or not videoOpts_.isValid()) {
initStream(videoCodec_);
initStream(videoCodec_, frame->hw_frames_ctx);
startIO();
} else {
return 0;
......
......@@ -110,8 +110,8 @@ private:
AVCodecContext* prepareEncoderContext(AVCodec* outputCodec, bool is_video);
void forcePresetX264(AVCodecContext* encoderCtx);
void extractProfileLevelID(const std::string &parameters, AVCodecContext *ctx);
int initStream(const std::string& codecName);
int initStream(const SystemCodecInfo& systemCodecInfo);
int initStream(const std::string& codecName, AVBufferRef* framesCtx);
int initStream(const SystemCodecInfo& systemCodecInfo, AVBufferRef* framesCtx);
void openIOContext();
void startIO();
......
......@@ -104,7 +104,7 @@ HardwareAccel::transfer(const VideoFrame& frame)
return nullptr;
}
return transferToMainMemory(frame, AV_PIX_FMT_NV12);
return transferToMainMemory(frame, swFormat_);
} else if (type_ == CODEC_ENCODER) {
auto input = frame.pointer();
if (input->format != swFormat_) {
......@@ -141,14 +141,14 @@ HardwareAccel::transfer(const VideoFrame& frame)
}
void
HardwareAccel::setDetails(AVCodecContext* codecCtx, AVDictionary** /*d*/)
HardwareAccel::setDetails(AVCodecContext* codecCtx)
{
if (type_ == CODEC_DECODER) {
codecCtx->hw_device_ctx = av_buffer_ref(deviceCtx_);
codecCtx->get_format = getFormatCb;
codecCtx->thread_safe_callbacks = 1;
} else if (type_ == CODEC_ENCODER) {
codecCtx->hw_device_ctx = av_buffer_ref(deviceCtx_);
// encoder doesn't need a device context, only a frame context
codecCtx->hw_frames_ctx = av_buffer_ref(framesCtx_);
}
}
......@@ -204,6 +204,28 @@ HardwareAccel::initFrame(int width, int height)
return ret >= 0;
}
bool
HardwareAccel::linkHardware(AVBufferRef* framesCtx)
{
if (framesCtx) {
// Force sw_format to match swFormat_. Frame is never transferred to main
// memory when hardware is linked, so the sw_format doesn't matter.
auto hw = reinterpret_cast<AVHWFramesContext*>(framesCtx->data);
hw->sw_format = swFormat_;
if (framesCtx_)
av_buffer_unref(&framesCtx_);
framesCtx_ = av_buffer_ref(framesCtx);
if ((linked_ = (framesCtx_ != nullptr))) {
RING_DBG() << "Hardware transcoding pipeline successfully set up for"
<< " encoder '" << getCodecName() << "'";
}
return linked_;
} else {
return false;
}
}
std::unique_ptr<VideoFrame>
HardwareAccel::transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat)
{
......@@ -232,7 +254,7 @@ HardwareAccel::transferToMainMemory(const VideoFrame& frame, AVPixelFormat desir
}
std::unique_ptr<HardwareAccel>
HardwareAccel::setupDecoder(AVCodecID id)
HardwareAccel::setupDecoder(AVCodecID id, int width, int height)
{
static const HardwareAPI apiList[] = {
{ "vaapi", AV_PIX_FMT_VAAPI, AV_PIX_FMT_NV12, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_VP8, AV_CODEC_ID_MJPEG } },
......@@ -243,7 +265,7 @@ HardwareAccel::setupDecoder(AVCodecID id)
for (const auto& api : apiList) {
if (std::find(api.supportedCodecs.begin(), api.supportedCodecs.end(), id) != api.supportedCodecs.end()) {
auto accel = std::make_unique<HardwareAccel>(id, api.name, api.format, api.swFormat, CODEC_DECODER);
if (accel->initDevice()) {
if (accel->initDevice() && accel->initFrame(width, height)) {
RING_DBG() << "Attempting to use hardware decoder " << accel->getCodecName() << " with " << api.name;
return accel;
}
......@@ -254,7 +276,7 @@ HardwareAccel::setupDecoder(AVCodecID id)
}
std::unique_ptr<HardwareAccel>
HardwareAccel::setupEncoder(AVCodecID id, int width, int height)
HardwareAccel::setupEncoder(AVCodecID id, int width, int height, AVBufferRef* framesCtx)
{
static const HardwareAPI apiList[] = {
{ "vaapi", AV_PIX_FMT_VAAPI, AV_PIX_FMT_NV12, { AV_CODEC_ID_H264, AV_CODEC_ID_MJPEG, AV_CODEC_ID_VP8 } },
......@@ -267,7 +289,9 @@ HardwareAccel::setupEncoder(AVCodecID id, int width, int height)
auto accel = std::make_unique<HardwareAccel>(id, api.name, api.format, api.swFormat, CODEC_ENCODER);
const auto& codecName = accel->getCodecName();
if (avcodec_find_encoder_by_name(codecName.c_str())) {
if (accel->initDevice() && accel->initFrame(width, height)) {
// Set up a fully accelerated pipeline, else fallback to using the main memory
if (accel->linkHardware(framesCtx)
|| (accel->initDevice() && accel->initFrame(width, height))) {
RING_DBG() << "Attempting to use hardware encoder " << codecName;
return accel;
}
......
......@@ -37,12 +37,13 @@ public:
/**
* Static factory method for hardware decoding.
*/
static std::unique_ptr<HardwareAccel> setupDecoder(AVCodecID id);
static std::unique_ptr<HardwareAccel> setupDecoder(AVCodecID id, int width, int height);
/**
* Static factory method for hardware encoding.
*/
static std::unique_ptr<HardwareAccel> setupEncoder(AVCodecID id, int width, int height);
static std::unique_ptr<HardwareAccel> setupEncoder(AVCodecID id, int width, int height,
AVBufferRef* framesCtx = nullptr);
/**
* Transfers a hardware decoded frame back to main memory. Should be called after
......@@ -86,19 +87,27 @@ public:
/**
* Gets the name of the codec.
* Decoding: equivalent to avcodec_get_name(id_)
* Decoding: avcodec_get_name(id_)
* Encoding: avcodec_get_name(id_) + '_' + name_
*/
std::string getCodecName() const;
/**
* Returns whether or not the decoder is linked to an encoder or vice-versa. Being linked
* means an encoder can directly use the decoder's hardware frame, without first
* transferring it to main memory.
*/
bool isLinked() const { return linked_; }
/**
* Set some extra details in the codec context. Should be called after a successful
* setup (setupDecoder or setupEncoder).
* For decoding, sets the hw_device_ctx and get_format callback. For encoding, sets
* hw_device_ctx and hw_frames_ctx, and may set some hardware specific options in
* the dictionary.
* For decoding, sets the hw_device_ctx and get_format callback. If the decoder has
* a frames context, mark as linked.
* For encoding, sets hw_device_ctx and hw_frames_ctx, and may set some hardware
* codec options.
*/
void setDetails(AVCodecContext* codecCtx, AVDictionary** d);
void setDetails(AVCodecContext* codecCtx);
/**
* Transfers a hardware decoded frame back to main memory. Should be called after
......@@ -110,6 +119,12 @@ public:
*/
std::unique_ptr<VideoFrame> transfer(const VideoFrame& frame);
/**
* Links this HardwareAccel's frames context with the passed in context. This serves
* to skip transferring a decoded frame back to main memory before encoding.
*/
bool linkHardware(AVBufferRef* framesCtx);
private:
bool initDevice();
bool initFrame(int width, int height);
......@@ -119,6 +134,7 @@ private:
AVPixelFormat format_ {AV_PIX_FMT_NONE};
AVPixelFormat swFormat_ {AV_PIX_FMT_NONE};
CodecType type_ {CODEC_NONE};
bool linked_ {false};
AVBufferRef* deviceCtx_ {nullptr};
AVBufferRef* framesCtx_ {nullptr};
......
......@@ -106,13 +106,7 @@ VideoSender::encodeAndSendVideo(VideoFrame& input_frame)
changeOrientationCallback_(rotation_);
}
#ifdef RING_ACCEL
auto framePtr = HardwareAccel::transferToMainMemory(input_frame, AV_PIX_FMT_NV12);
auto& swFrame = *framePtr;
#else
auto& swFrame = input_frame;
#endif
if (videoEncoder_->encode(swFrame, is_keyframe, frameNumber_++) < 0)
if (videoEncoder_->encode(input_frame, is_keyframe, frameNumber_++) < 0)
RING_ERR("encoding failed");
}
#ifdef DEBUG_SDP
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment