diff --git a/src/media/media_codec.h b/src/media/media_codec.h index be4467e3d1de5e97fb19f11928f5dae88350e1a7..13924711a4368fa2cce659b2324160aa34f05168 100644 --- a/src/media/media_codec.h +++ b/src/media/media_codec.h @@ -279,6 +279,7 @@ struct MediaDescription { /** Video parameters */ std::string parameters {}; bool auto_quality {false}; + bool linkableHW {false}; /** Crypto parameters */ CryptoAttribute crypto {}; diff --git a/src/media/media_encoder.cpp b/src/media/media_encoder.cpp index f5235143ce9af7990f4248e210309aa2f7161213..a12242528fbfd0f749461f1161baf9441985091c 100644 --- a/src/media/media_encoder.cpp +++ b/src/media/media_encoder.cpp @@ -116,6 +116,7 @@ MediaEncoder::setOptions(const MediaDescription& args) libav_utils::setDictValue(&options_, "parameters", args.parameters); auto_quality = args.auto_quality; + linkableHW_ = args.linkableHW; } void @@ -310,8 +311,11 @@ MediaEncoder::encode(VideoFrame& input, bool is_keyframe, int64_t frame_number) } else { #else std::unique_ptr<VideoFrame> framePtr; - if (accel_ && accel_->isLinked()) { + if (accel_ && accel_->isLinked() && isHardware) { // Fully accelerated pipeline, skip main memory + // We have to check if the frame is hardware even if + // we are using linked HW encoder and decoder because after + // conference mixing the frame become software (prevent crashes) frame = input.pointer(); } else if (isHardware) { // Hardware decoded frame, transfer back to main memory @@ -655,7 +659,7 @@ MediaEncoder::initCodec(AVMediaType mediaType, AVCodecID avcodecId, AVBufferRef* if (enableAccel_) { if (accel_ = video::HardwareAccel::setupEncoder( static_cast<AVCodecID>(avcodecId), - videoOpts_.width, videoOpts_.height, framesCtx)) { + videoOpts_.width, videoOpts_.height, linkableHW_, framesCtx)) { outputCodec_ = avcodec_find_encoder_by_name(accel_->getCodecName().c_str()); } } else { diff --git a/src/media/media_encoder.h b/src/media/media_encoder.h index 0b978589aeb2fef99a97b471bffeae586aed4e8f..d5dd8bc8483e5d36ad7ffd50ce12071f375c3806 100644 --- a/src/media/media_encoder.h +++ b/src/media/media_encoder.h @@ -133,6 +133,7 @@ private: AVCodec* outputCodec_ = nullptr; std::mutex encMutex_; bool auto_quality {false}; + bool linkableHW_ {false}; void initH264(AVCodecContext* encoderCtx, uint64_t br); void initVP8(AVCodecContext* encoderCtx, uint64_t br); diff --git a/src/media/video/accel.cpp b/src/media/video/accel.cpp index 88033142da3398f192a12b1440ba96e9c769cb31..bdc69a97bd7b76df1c3b732c82f76745c75e687d 100644 --- a/src/media/video/accel.cpp +++ b/src/media/video/accel.cpp @@ -155,7 +155,10 @@ HardwareAccel::setDetails(AVCodecContext* codecCtx) bool HardwareAccel::initDevice() { - return av_hwdevice_ctx_create(&deviceCtx_, hwType_, nullptr, nullptr, 0) >= 0; + int ret = av_hwdevice_ctx_create(&deviceCtx_, hwType_, nullptr, nullptr, 0); + if (ret < 0) + JAMI_ERR("Creating hardware device context failed: %s (%d)", libav_utils::getError(ret).c_str(), ret); + return ret >= 0; } bool @@ -178,8 +181,10 @@ HardwareAccel::initFrame(int width, int height) ctx->height = height; ctx->initial_pool_size = 20; // TODO try other values - if ((ret = av_hwframe_ctx_init(framesCtx_)) < 0) + if ((ret = av_hwframe_ctx_init(framesCtx_)) < 0) { + JAMI_ERR("Failed to initialize hardware frame context: %s (%d)", libav_utils::getError(ret).c_str(), ret); av_buffer_unref(&framesCtx_); + } return ret >= 0; } @@ -262,7 +267,7 @@ HardwareAccel::setupDecoder(AVCodecID id, int width, int height) } std::unique_ptr<HardwareAccel> -HardwareAccel::setupEncoder(AVCodecID id, int width, int height, AVBufferRef* framesCtx) +HardwareAccel::setupEncoder(AVCodecID id, int width, int height, bool linkable, AVBufferRef* framesCtx) { static const HardwareAPI apiList[] = { { "nvenc", AV_HWDEVICE_TYPE_CUDA, AV_PIX_FMT_CUDA, AV_PIX_FMT_NV12, { AV_CODEC_ID_H264, AV_CODEC_ID_H265 } }, @@ -278,10 +283,12 @@ HardwareAccel::setupEncoder(AVCodecID id, int width, int height, AVBufferRef* fr const auto& codecName = accel->getCodecName(); if (avcodec_find_encoder_by_name(codecName.c_str())) { if (accel->initDevice()) { + bool link = false; + if (linkable) + link = accel->linkHardware(framesCtx); // we don't need frame context for videotoolbox if (api.format == AV_PIX_FMT_VIDEOTOOLBOX || - accel->linkHardware(framesCtx) || - accel->initFrame(width, height)) { + link || accel->initFrame(width, height)) { JAMI_DBG() << "Attempting to use hardware encoder " << codecName << " with " << api.name; return accel; } diff --git a/src/media/video/accel.h b/src/media/video/accel.h index d9e777bb14639412fb8bb3598ab4a03f239ec284..b18b195cf5c188c1dc659d8b123d12dc06156bb3 100644 --- a/src/media/video/accel.h +++ b/src/media/video/accel.h @@ -46,7 +46,7 @@ public: /** * @brief Static factory method for hardware encoding. */ - static std::unique_ptr<HardwareAccel> setupEncoder(AVCodecID id, int width, int height, + static std::unique_ptr<HardwareAccel> setupEncoder(AVCodecID id, int width, int height, bool linkable, AVBufferRef* framesCtx = nullptr); /** diff --git a/src/media/video/video_rtp_session.cpp b/src/media/video/video_rtp_session.cpp index 923675b09911d22bbaf22579b451da516f2611b2..b1e8cce04f5e6da37a035c08bc9a62ed1583a99a 100644 --- a/src/media/video/video_rtp_session.cpp +++ b/src/media/video/video_rtp_session.cpp @@ -133,6 +133,7 @@ void VideoRtpSession::startSender() auto autoQuality = codecVideo->isAutoQualityEnabled; send_.auto_quality = autoQuality; + send_.linkableHW = conference_ == nullptr; if (sender_) initSeqVal_ = sender_->getLastSeqValue() + 10; // Skip a few sequences to make nvenc happy on a sender restart @@ -326,6 +327,10 @@ VideoRtpSession::enterConference(Conference* conference) videoMixer_->setParameters(localVideoParams_.width, localVideoParams_.height); #endif setupConferenceVideoPipeline(*conference_); + + // Restart encoder with conference parameter ON in order to unlink HW encoder + // from HW decoder. + restartSender(); } }