accel: fix crash during conferences with hardware acceleration

NVENC is still limited to 2 concurrent encoding sessions for most of nvidia GPU

Change-Id: Iad55018543ec316051639f489d4189534fef22f7

src/media/media_codec.h

@@ -279,6 +279,7 @@ struct MediaDescription {
     /** Video parameters */
     std::string parameters {};
     bool auto_quality {false};
+    bool linkableHW {false};
 
     /** Crypto parameters */
     CryptoAttribute crypto {};

src/media/media_encoder.cpp

@@ -116,6 +116,7 @@ MediaEncoder::setOptions(const MediaDescription& args)
         libav_utils::setDictValue(&options_, "parameters", args.parameters);
 
     auto_quality = args.auto_quality;
+    linkableHW_ = args.linkableHW;
 }
 
 void
@@ -310,8 +311,11 @@ MediaEncoder::encode(VideoFrame& input, bool is_keyframe, int64_t frame_number)
     } else {
 #else
     std::unique_ptr<VideoFrame> framePtr;
-    if (accel_ && accel_->isLinked()) {
+    if (accel_ && accel_->isLinked() && isHardware) {
         // Fully accelerated pipeline, skip main memory
+        // We have to check if the frame is hardware even if
+        // we are using linked HW encoder and decoder because after
+        // conference mixing the frame become software (prevent crashes)
         frame = input.pointer();
     } else if (isHardware) {
         // Hardware decoded frame, transfer back to main memory
@@ -655,7 +659,7 @@ MediaEncoder::initCodec(AVMediaType mediaType, AVCodecID avcodecId, AVBufferRef*
         if (enableAccel_) {
             if (accel_ = video::HardwareAccel::setupEncoder(
                 static_cast<AVCodecID>(avcodecId),
-                videoOpts_.width, videoOpts_.height, framesCtx)) {
+                videoOpts_.width, videoOpts_.height, linkableHW_, framesCtx)) {
                 outputCodec_ = avcodec_find_encoder_by_name(accel_->getCodecName().c_str());
             }
         } else {

src/media/media_encoder.h

@@ -133,6 +133,7 @@ private:
     AVCodec* outputCodec_ = nullptr;
     std::mutex encMutex_;
     bool auto_quality {false};
+    bool linkableHW_ {false};
 
     void initH264(AVCodecContext* encoderCtx, uint64_t br);
     void initVP8(AVCodecContext* encoderCtx, uint64_t br);

src/media/video/accel.cpp

@@ -155,7 +155,10 @@ HardwareAccel::setDetails(AVCodecContext* codecCtx)
 bool
 HardwareAccel::initDevice()
 {
-    return av_hwdevice_ctx_create(&deviceCtx_, hwType_, nullptr, nullptr, 0) >= 0;
+    int ret = av_hwdevice_ctx_create(&deviceCtx_, hwType_, nullptr, nullptr, 0);
+    if (ret < 0)
+        JAMI_ERR("Creating hardware device context failed: %s (%d)", libav_utils::getError(ret).c_str(), ret);
+    return ret >= 0;
 }
 
 bool
@@ -178,8 +181,10 @@ HardwareAccel::initFrame(int width, int height)
     ctx->height = height;
     ctx->initial_pool_size = 20; // TODO try other values
 
-    if ((ret = av_hwframe_ctx_init(framesCtx_)) < 0)
+    if ((ret = av_hwframe_ctx_init(framesCtx_)) < 0) {
+        JAMI_ERR("Failed to initialize hardware frame context: %s (%d)", libav_utils::getError(ret).c_str(), ret);
         av_buffer_unref(&framesCtx_);
+    }
 
     return ret >= 0;
 }
@@ -262,7 +267,7 @@ HardwareAccel::setupDecoder(AVCodecID id, int width, int height)
 }
 
 std::unique_ptr<HardwareAccel>
-HardwareAccel::setupEncoder(AVCodecID id, int width, int height, AVBufferRef* framesCtx)
+HardwareAccel::setupEncoder(AVCodecID id, int width, int height, bool linkable, AVBufferRef* framesCtx)
 {
     static const HardwareAPI apiList[] = {
         { "nvenc", AV_HWDEVICE_TYPE_CUDA, AV_PIX_FMT_CUDA, AV_PIX_FMT_NV12, { AV_CODEC_ID_H264, AV_CODEC_ID_H265 } },
@@ -278,10 +283,12 @@ HardwareAccel::setupEncoder(AVCodecID id, int width, int height, AVBufferRef* fr
             const auto& codecName = accel->getCodecName();
             if (avcodec_find_encoder_by_name(codecName.c_str())) {
                 if (accel->initDevice()) {
+                    bool link = false;
+                    if (linkable)
+                        link = accel->linkHardware(framesCtx);
                     // we don't need frame context for videotoolbox
                     if (api.format == AV_PIX_FMT_VIDEOTOOLBOX ||
-                        accel->linkHardware(framesCtx) ||
-                        accel->initFrame(width, height)) {
+                        link || accel->initFrame(width, height)) {
                         JAMI_DBG() << "Attempting to use hardware encoder " << codecName << " with " << api.name;
                         return accel;
                     }

src/media/video/accel.h

@@ -46,7 +46,7 @@ public:
     /**
      * @brief Static factory method for hardware encoding.
      */
-    static std::unique_ptr<HardwareAccel> setupEncoder(AVCodecID id, int width, int height,
+    static std::unique_ptr<HardwareAccel> setupEncoder(AVCodecID id, int width, int height, bool linkable,
         AVBufferRef* framesCtx = nullptr);
 
     /**

src/media/video/video_rtp_session.cpp

@@ -133,6 +133,7 @@ void VideoRtpSession::startSender()
         auto autoQuality = codecVideo->isAutoQualityEnabled;
 
         send_.auto_quality = autoQuality;
+        send_.linkableHW = conference_ == nullptr;
 
         if (sender_)
             initSeqVal_ = sender_->getLastSeqValue() + 10; // Skip a few sequences to make nvenc happy on a sender restart
@@ -326,6 +327,10 @@ VideoRtpSession::enterConference(Conference* conference)
         videoMixer_->setParameters(localVideoParams_.width, localVideoParams_.height);
 #endif
         setupConferenceVideoPipeline(*conference_);
+
+        // Restart encoder with conference parameter ON in order to unlink HW encoder
+        // from HW decoder.
+        restartSender();
     }
 }