diff --git a/src/media/media_decoder.cpp b/src/media/media_decoder.cpp index 40e2f05b410d14d7fa522535fe5140257ec182c4..b3d63d6b44449d70ebaafba6a53feea3eccd081f 100644 --- a/src/media/media_decoder.cpp +++ b/src/media/media_decoder.cpp @@ -227,9 +227,10 @@ MediaDecoder::setupStream(AVMediaType mediaType) #ifdef RING_ACCEL if (mediaType == AVMEDIA_TYPE_VIDEO) { if (enableAccel_) { - accel_ = video::HardwareAccel::setupDecoder(decoderCtx_->codec_id); + accel_ = video::HardwareAccel::setupDecoder(decoderCtx_->codec_id, + decoderCtx_->width, decoderCtx_->height); if (accel_) { - accel_->setDetails(decoderCtx_, &options_); + accel_->setDetails(decoderCtx_); decoderCtx_->opaque = accel_.get(); } } else if (Manager::instance().videoPreferences.getDecodingAccelerated()) { diff --git a/src/media/media_encoder.cpp b/src/media/media_encoder.cpp index 5991909b1c7ef9b7eb2adb4e81faff01fc60ba4b..7d3a9be176577371a4bb92ee5c52b5bb197b4c5b 100644 --- a/src/media/media_encoder.cpp +++ b/src/media/media_encoder.cpp @@ -162,7 +162,7 @@ MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo) { if (systemCodecInfo.mediaType == MEDIA_AUDIO) { audioCodec_ = systemCodecInfo.name; - return initStream(systemCodecInfo); + return initStream(systemCodecInfo, nullptr); } else { videoCodec_ = systemCodecInfo.name; // TODO only support 1 audio stream and 1 video stream per encoder @@ -174,17 +174,17 @@ MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo) } int -MediaEncoder::initStream(const std::string& codecName) +MediaEncoder::initStream(const std::string& codecName, AVBufferRef* framesCtx) { const auto codecInfo = getSystemCodecContainer()->searchCodecByName(codecName, MEDIA_ALL); if (codecInfo) - return initStream(*codecInfo); + return initStream(*codecInfo, framesCtx); else return -1; } int -MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo) +MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo, AVBufferRef* framesCtx) { AVCodec* outputCodec = nullptr; AVCodecContext* encoderCtx = nullptr; @@ -192,7 +192,8 @@ MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo) if (systemCodecInfo.mediaType == MEDIA_VIDEO) { if (enableAccel_) { if (accel_ = video::HardwareAccel::setupEncoder( - static_cast<AVCodecID>(systemCodecInfo.avcodecId), videoOpts_.width, videoOpts_.height)) { + static_cast<AVCodecID>(systemCodecInfo.avcodecId), + videoOpts_.width, videoOpts_.height, framesCtx)) { outputCodec = avcodec_find_encoder_by_name(accel_->getCodecName().c_str()); } } else { @@ -221,7 +222,7 @@ MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo) #ifdef RING_ACCEL if (accel_) { - accel_->setDetails(encoderCtx, &options_); + accel_->setDetails(encoderCtx); encoderCtx->opaque = accel_.get(); } #endif @@ -383,28 +384,49 @@ MediaEncoder::encode(VideoFrame& input, bool is_keyframe, int64_t frame_number) { if (!initialized_) { - initStream(videoCodec_); + initStream(videoCodec_, input.pointer()->hw_frames_ctx); startIO(); } - /* Prepare a frame suitable to our encoder frame format, - * keeping also the input aspect ratio. - */ - libav_utils::fillWithBlack(scaledFrame_.pointer()); - - scaler_.scale_with_aspect(input, scaledFrame_); - - // Copy frame so the VideoScaler can still use the software frame (input) - VideoFrame copy; - copy.copyFrom(scaledFrame_); + AVFrame* frame; +#ifdef RING_ACCEL + auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input.format())); + bool isHardware = desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL); + std::unique_ptr<VideoFrame> framePtr; + if (accel_ && accel_->isLinked()) { + // Fully accelerated pipeline, skip main memory + frame = input.pointer(); + } else if (isHardware) { + // Hardware decoded frame, transfer back to main memory + // Transfer to GPU if we have a hardware encoder + AVPixelFormat pix = (accel_ ? accel_->getSoftwareFormat() : AV_PIX_FMT_YUV420P); + framePtr = video::HardwareAccel::transferToMainMemory(input, pix); + if (accel_) + framePtr = accel_->transfer(*framePtr); + frame = framePtr->pointer(); + } else if (accel_) { + // Software decoded frame with a hardware encoder, convert to accepted format first + auto pix = accel_->getSoftwareFormat(); + if (input.format() != pix) { + framePtr = scaler_.convertFormat(input, pix); + framePtr = accel_->transfer(*framePtr); + } else { + framePtr = accel_->transfer(input); + } + frame = framePtr->pointer(); + } else { +#endif + libav_utils::fillWithBlack(scaledFrame_.pointer()); + scaler_.scale_with_aspect(input, scaledFrame_); + frame = scaledFrame_.pointer(); +#ifdef RING_ACCEL + } +#endif - auto frame = copy.pointer(); AVCodecContext* enc = encoders_[currentStreamIdx_]; - // ideally, time base is the inverse of framerate, but this may not always be the case - if (enc->framerate.num == enc->time_base.den && enc->framerate.den == enc->time_base.num) - frame->pts = frame_number; - else - frame->pts = (frame_number / (rational<int64_t>(enc->framerate) * rational<int64_t>(enc->time_base))).real<int64_t>(); + frame->pts = frame_number; + if (enc->framerate.num != enc->time_base.den || enc->framerate.den != enc->time_base.num) + frame->pts /= (rational<int64_t>(enc->framerate) * rational<int64_t>(enc->time_base)).real<int64_t>(); if (is_keyframe) { frame->pict_type = AV_PICTURE_TYPE_I; @@ -414,19 +436,6 @@ MediaEncoder::encode(VideoFrame& input, bool is_keyframe, frame->key_frame = 0; } -#ifdef RING_ACCEL - // NOTE needs to be at same scope as call to encode - std::unique_ptr<VideoFrame> framePtr; - if (accel_) { - framePtr = accel_->transfer(copy); - if (!framePtr) { - RING_ERR() << "Hardware encoding failure"; - return -1; - } - frame = framePtr->pointer(); - } -#endif - return encode(frame, currentStreamIdx_); } #endif // RING_VIDEO @@ -454,7 +463,7 @@ MediaEncoder::encode(AVFrame* frame, int streamIdx) // Initialize on first video frame, or first audio frame if no video stream bool isVideo = (frame->width > 0 && frame->height > 0); if (isVideo or not videoOpts_.isValid()) { - initStream(videoCodec_); + initStream(videoCodec_, frame->hw_frames_ctx); startIO(); } else { return 0; diff --git a/src/media/media_encoder.h b/src/media/media_encoder.h index 0c253543225d9ebe280c53b98a227aa51a665101..aca43124149b21adad2ba5d044ebcf9277b80fd6 100644 --- a/src/media/media_encoder.h +++ b/src/media/media_encoder.h @@ -110,8 +110,8 @@ private: AVCodecContext* prepareEncoderContext(AVCodec* outputCodec, bool is_video); void forcePresetX264(AVCodecContext* encoderCtx); void extractProfileLevelID(const std::string ¶meters, AVCodecContext *ctx); - int initStream(const std::string& codecName); - int initStream(const SystemCodecInfo& systemCodecInfo); + int initStream(const std::string& codecName, AVBufferRef* framesCtx); + int initStream(const SystemCodecInfo& systemCodecInfo, AVBufferRef* framesCtx); void openIOContext(); void startIO(); diff --git a/src/media/video/accel.cpp b/src/media/video/accel.cpp index 1e7e7216c7bc38dbe3ba0d72d5d70b2f8fff4400..fd4e7ab320e25218ce145d1e1b74f2e033e4d920 100644 --- a/src/media/video/accel.cpp +++ b/src/media/video/accel.cpp @@ -104,7 +104,7 @@ HardwareAccel::transfer(const VideoFrame& frame) return nullptr; } - return transferToMainMemory(frame, AV_PIX_FMT_NV12); + return transferToMainMemory(frame, swFormat_); } else if (type_ == CODEC_ENCODER) { auto input = frame.pointer(); if (input->format != swFormat_) { @@ -141,14 +141,14 @@ HardwareAccel::transfer(const VideoFrame& frame) } void -HardwareAccel::setDetails(AVCodecContext* codecCtx, AVDictionary** /*d*/) +HardwareAccel::setDetails(AVCodecContext* codecCtx) { if (type_ == CODEC_DECODER) { codecCtx->hw_device_ctx = av_buffer_ref(deviceCtx_); codecCtx->get_format = getFormatCb; codecCtx->thread_safe_callbacks = 1; } else if (type_ == CODEC_ENCODER) { - codecCtx->hw_device_ctx = av_buffer_ref(deviceCtx_); + // encoder doesn't need a device context, only a frame context codecCtx->hw_frames_ctx = av_buffer_ref(framesCtx_); } } @@ -204,6 +204,28 @@ HardwareAccel::initFrame(int width, int height) return ret >= 0; } +bool +HardwareAccel::linkHardware(AVBufferRef* framesCtx) +{ + if (framesCtx) { + // Force sw_format to match swFormat_. Frame is never transferred to main + // memory when hardware is linked, so the sw_format doesn't matter. + auto hw = reinterpret_cast<AVHWFramesContext*>(framesCtx->data); + hw->sw_format = swFormat_; + + if (framesCtx_) + av_buffer_unref(&framesCtx_); + framesCtx_ = av_buffer_ref(framesCtx); + if ((linked_ = (framesCtx_ != nullptr))) { + RING_DBG() << "Hardware transcoding pipeline successfully set up for" + << " encoder '" << getCodecName() << "'"; + } + return linked_; + } else { + return false; + } +} + std::unique_ptr<VideoFrame> HardwareAccel::transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat) { @@ -232,7 +254,7 @@ HardwareAccel::transferToMainMemory(const VideoFrame& frame, AVPixelFormat desir } std::unique_ptr<HardwareAccel> -HardwareAccel::setupDecoder(AVCodecID id) +HardwareAccel::setupDecoder(AVCodecID id, int width, int height) { static const HardwareAPI apiList[] = { { "vaapi", AV_PIX_FMT_VAAPI, AV_PIX_FMT_NV12, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_VP8, AV_CODEC_ID_MJPEG } }, @@ -243,7 +265,7 @@ HardwareAccel::setupDecoder(AVCodecID id) for (const auto& api : apiList) { if (std::find(api.supportedCodecs.begin(), api.supportedCodecs.end(), id) != api.supportedCodecs.end()) { auto accel = std::make_unique<HardwareAccel>(id, api.name, api.format, api.swFormat, CODEC_DECODER); - if (accel->initDevice()) { + if (accel->initDevice() && accel->initFrame(width, height)) { RING_DBG() << "Attempting to use hardware decoder " << accel->getCodecName() << " with " << api.name; return accel; } @@ -254,7 +276,7 @@ HardwareAccel::setupDecoder(AVCodecID id) } std::unique_ptr<HardwareAccel> -HardwareAccel::setupEncoder(AVCodecID id, int width, int height) +HardwareAccel::setupEncoder(AVCodecID id, int width, int height, AVBufferRef* framesCtx) { static const HardwareAPI apiList[] = { { "vaapi", AV_PIX_FMT_VAAPI, AV_PIX_FMT_NV12, { AV_CODEC_ID_H264, AV_CODEC_ID_MJPEG, AV_CODEC_ID_VP8 } }, @@ -267,7 +289,9 @@ HardwareAccel::setupEncoder(AVCodecID id, int width, int height) auto accel = std::make_unique<HardwareAccel>(id, api.name, api.format, api.swFormat, CODEC_ENCODER); const auto& codecName = accel->getCodecName(); if (avcodec_find_encoder_by_name(codecName.c_str())) { - if (accel->initDevice() && accel->initFrame(width, height)) { + // Set up a fully accelerated pipeline, else fallback to using the main memory + if (accel->linkHardware(framesCtx) + || (accel->initDevice() && accel->initFrame(width, height))) { RING_DBG() << "Attempting to use hardware encoder " << codecName; return accel; } diff --git a/src/media/video/accel.h b/src/media/video/accel.h index 85fdbe88c620ee37bf6e8b7e60f280512c4609b5..b6f1a7f7c9f498538483f268be4c03ac47f85672 100644 --- a/src/media/video/accel.h +++ b/src/media/video/accel.h @@ -37,12 +37,13 @@ public: /** * Static factory method for hardware decoding. */ - static std::unique_ptr<HardwareAccel> setupDecoder(AVCodecID id); + static std::unique_ptr<HardwareAccel> setupDecoder(AVCodecID id, int width, int height); /** * Static factory method for hardware encoding. */ - static std::unique_ptr<HardwareAccel> setupEncoder(AVCodecID id, int width, int height); + static std::unique_ptr<HardwareAccel> setupEncoder(AVCodecID id, int width, int height, + AVBufferRef* framesCtx = nullptr); /** * Transfers a hardware decoded frame back to main memory. Should be called after @@ -86,19 +87,27 @@ public: /** * Gets the name of the codec. - * Decoding: equivalent to avcodec_get_name(id_) + * Decoding: avcodec_get_name(id_) * Encoding: avcodec_get_name(id_) + '_' + name_ */ std::string getCodecName() const; + /** + * Returns whether or not the decoder is linked to an encoder or vice-versa. Being linked + * means an encoder can directly use the decoder's hardware frame, without first + * transferring it to main memory. + */ + bool isLinked() const { return linked_; } + /** * Set some extra details in the codec context. Should be called after a successful * setup (setupDecoder or setupEncoder). - * For decoding, sets the hw_device_ctx and get_format callback. For encoding, sets - * hw_device_ctx and hw_frames_ctx, and may set some hardware specific options in - * the dictionary. + * For decoding, sets the hw_device_ctx and get_format callback. If the decoder has + * a frames context, mark as linked. + * For encoding, sets hw_device_ctx and hw_frames_ctx, and may set some hardware + * codec options. */ - void setDetails(AVCodecContext* codecCtx, AVDictionary** d); + void setDetails(AVCodecContext* codecCtx); /** * Transfers a hardware decoded frame back to main memory. Should be called after @@ -110,6 +119,12 @@ public: */ std::unique_ptr<VideoFrame> transfer(const VideoFrame& frame); + /** + * Links this HardwareAccel's frames context with the passed in context. This serves + * to skip transferring a decoded frame back to main memory before encoding. + */ + bool linkHardware(AVBufferRef* framesCtx); + private: bool initDevice(); bool initFrame(int width, int height); @@ -119,6 +134,7 @@ private: AVPixelFormat format_ {AV_PIX_FMT_NONE}; AVPixelFormat swFormat_ {AV_PIX_FMT_NONE}; CodecType type_ {CODEC_NONE}; + bool linked_ {false}; AVBufferRef* deviceCtx_ {nullptr}; AVBufferRef* framesCtx_ {nullptr}; diff --git a/src/media/video/video_sender.cpp b/src/media/video/video_sender.cpp index 205b24c1fc869546065f46c71fab477fcdba5d51..ca47036c9b927133ee7492dfc81a8c3c056dd724 100644 --- a/src/media/video/video_sender.cpp +++ b/src/media/video/video_sender.cpp @@ -106,13 +106,7 @@ VideoSender::encodeAndSendVideo(VideoFrame& input_frame) changeOrientationCallback_(rotation_); } -#ifdef RING_ACCEL - auto framePtr = HardwareAccel::transferToMainMemory(input_frame, AV_PIX_FMT_NV12); - auto& swFrame = *framePtr; -#else - auto& swFrame = input_frame; -#endif - if (videoEncoder_->encode(swFrame, is_keyframe, frameNumber_++) < 0) + if (videoEncoder_->encode(input_frame, is_keyframe, frameNumber_++) < 0) RING_ERR("encoding failed"); } #ifdef DEBUG_SDP