diff --git a/src/media/video/sinkclient.cpp b/src/media/video/sinkclient.cpp
index 07cf155cce3c7d5a6af9af2a27904a88047e945c..6d8ba4b65b80f1d4396724d741c5a8f6249bb078 100644
--- a/src/media/video/sinkclient.cpp
+++ b/src/media/video/sinkclient.cpp
@@ -336,13 +336,88 @@ SinkClient::SinkClient(const std::string& id, bool mixer)
JAMI_DBG("[Sink:%p] Sink [%s] created", this, getId().c_str());
}
+void
+SinkClient::sendFrameDirect(const std::shared_ptr<jami::MediaFrame>& frame_p)
+{
+ notify(frame_p);
+
+ DRing::FrameBuffer outFrame(av_frame_alloc());
+ av_frame_ref(outFrame.get(), std::static_pointer_cast<VideoFrame>(frame_p)->pointer());
+ if (crop_.w || crop_.h) {
+ outFrame->crop_top = crop_.y;
+ outFrame->crop_bottom = (size_t) outFrame->height - crop_.y - crop_.h;
+ outFrame->crop_left = crop_.x;
+ outFrame->crop_right = (size_t) outFrame->width - crop_.x - crop_.w;
+ av_frame_apply_cropping(outFrame.get(), AV_FRAME_CROP_UNALIGNED);
+ }
+ if (outFrame->height != height_ || outFrame->width != width_) {
+ setFrameSize(outFrame->width, outFrame->height);
+ return;
+ }
+ target_.push(std::move(outFrame));
+}
+
+void
+SinkClient::sendFrameTransformed(AVFrame* frame)
+{
+ if (frame->width > 0 and frame->height > 0) {
+ if (auto buffer_ptr = target_.pull()) {
+ scaler_->scale(frame, buffer_ptr.get());
+ target_.push(std::move(buffer_ptr));
+ }
+ }
+}
+
+std::shared_ptr<VideoFrame>
+SinkClient::applyTransform(VideoFrame& frame_p)
+{
+ std::shared_ptr<VideoFrame> frame = std::make_shared<VideoFrame>();
+#ifdef RING_ACCEL
+ auto desc = av_pix_fmt_desc_get((AVPixelFormat)frame_p.format());
+ if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
+ try {
+ frame = HardwareAccel::transferToMainMemory(frame_p, AV_PIX_FMT_NV12);
+ } catch (const std::runtime_error& e) {
+ JAMI_ERR("[Sink:%p] Transfert to hardware acceleration memory failed: %s",
+ this,
+ e.what());
+ return {};
+ }
+ } else
+#endif
+ frame->copyFrom(frame_p);
+
+ int angle = frame->getOrientation();
+ if (angle != rotation_) {
+ filter_ = getTransposeFilter(angle,
+ FILTER_INPUT_NAME,
+ frame->width(),
+ frame->height(),
+ frame->format(),
+ false);
+ rotation_ = angle;
+ }
+ if (filter_) {
+ filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME);
+ frame = std::static_pointer_cast<VideoFrame>(std::shared_ptr<MediaFrame>(filter_->readOutput()));
+ }
+ if (crop_.w || crop_.h) {
+ frame->pointer()->crop_top = crop_.y;
+ frame->pointer()->crop_bottom = (size_t) frame->height() - crop_.y - crop_.h;
+ frame->pointer()->crop_left = crop_.x;
+ frame->pointer()->crop_right = (size_t) frame->width() - crop_.x - crop_.w;
+ av_frame_apply_cropping(frame->pointer(), 0);
+ }
+ return frame;
+}
+
void
SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
const std::shared_ptr<MediaFrame>& frame_p)
{
#ifdef DEBUG_FPS
auto currentTime = std::chrono::system_clock::now();
- const std::chrono::duration<double> seconds = currentTime - lastFrameDebug_;
+ std::chrono::duration<double> seconds = currentTime - lastFrameDebug_;
++frameCount_;
if (seconds.count() > 1) {
auto fps = frameCount_ / seconds.count();
@@ -354,75 +429,26 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
#endif
std::unique_lock<std::mutex> lock(mtx_);
- if (target_.push and not target_.pull) {
- VideoFrame outFrame;
- outFrame.copyFrom(*std::static_pointer_cast<VideoFrame>(frame_p));
- if (crop_.w || crop_.h) {
- outFrame.pointer()->crop_top = crop_.y;
- outFrame.pointer()->crop_bottom = (size_t) outFrame.height() - crop_.y - crop_.h;
- outFrame.pointer()->crop_left = crop_.x;
- outFrame.pointer()->crop_right = (size_t) outFrame.width() - crop_.x - crop_.w;
- av_frame_apply_cropping(outFrame.pointer(), AV_FRAME_CROP_UNALIGNED);
- }
- if (outFrame.height() != height_ || outFrame.width() != width_) {
- setFrameSize(outFrame.width(), outFrame.height());
- return;
- }
- notify(std::static_pointer_cast<MediaFrame>(frame_p));
- target_.push(outFrame.getFrame());
+ bool hasObservers = getObserversCount() != 0;
+ bool hasDirectListener = target_.push and not target_.pull;
+ bool hasTransformedListener = target_.push and target_.pull;
+
+ if (hasDirectListener) {
+ sendFrameDirect(frame_p);
return;
}
- bool doTransfer = (target_.pull != nullptr);
+ bool doTransfer = hasTransformedListener or hasObservers;
#if HAVE_SHM
doTransfer |= (shm_ && doShmTransfer_);
#endif
if (doTransfer) {
- std::shared_ptr<VideoFrame> frame = std::make_shared<VideoFrame>();
-#ifdef RING_ACCEL
- auto desc = av_pix_fmt_desc_get(
- (AVPixelFormat)(std::static_pointer_cast<VideoFrame>(frame_p))->format());
- if (desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
- try {
- frame = HardwareAccel::transferToMainMemory(*std::static_pointer_cast<VideoFrame>(
- frame_p),
- AV_PIX_FMT_NV12);
- } catch (const std::runtime_error& e) {
- JAMI_ERR("[Sink:%p] Transfert to hardware acceleration memory failed: %s",
- this,
- e.what());
- return;
- }
- } else
-#endif
- frame->copyFrom(*std::static_pointer_cast<VideoFrame>(frame_p));
-
- int angle = frame->getOrientation();
-
- if (angle != rotation_) {
- filter_ = getTransposeFilter(angle,
- FILTER_INPUT_NAME,
- frame->width(),
- frame->height(),
- frame->format(),
- false);
- rotation_ = angle;
- }
- if (filter_) {
- filter_->feedInput(frame->pointer(), FILTER_INPUT_NAME);
- frame = std::static_pointer_cast<VideoFrame>(
- std::shared_ptr<MediaFrame>(filter_->readOutput()));
- }
- notify(std::static_pointer_cast<MediaFrame>(frame));
+ auto frame = applyTransform(*std::static_pointer_cast<VideoFrame>(frame_p));
+ if (not frame)
+ return;
- if (crop_.w || crop_.h) {
- frame->pointer()->crop_top = crop_.y;
- frame->pointer()->crop_bottom = (size_t) frame->height() - crop_.y - crop_.h;
- frame->pointer()->crop_left = crop_.x;
- frame->pointer()->crop_right = (size_t) frame->width() - crop_.x - crop_.w;
- av_frame_apply_cropping(frame->pointer(), AV_FRAME_CROP_UNALIGNED);
- }
+ notify(std::static_pointer_cast<MediaFrame>(frame));
if (frame->height() != height_ || frame->width() != width_) {
lock.unlock();
@@ -433,16 +459,8 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
if (shm_ && doShmTransfer_)
shm_->renderFrame(*frame);
#endif
- if (target_.pull) {
- int width = frame->width();
- int height = frame->height();
- if (width > 0 && height > 0) {
- if (auto buffer_ptr = target_.pull()) {
- scaler_->scale(*frame, buffer_ptr.get());
- target_.push(std::move(buffer_ptr));
- }
- }
- }
+ if (hasTransformedListener)
+ sendFrameTransformed(frame->pointer());
}
}
diff --git a/src/media/video/sinkclient.h b/src/media/video/sinkclient.h
index 4eee5523a62f8787e47a31a1a9bcf867f7c840ef..90313f487844e65034105ea309f62d406c2ef0f0 100644
--- a/src/media/video/sinkclient.h
+++ b/src/media/video/sinkclient.h
@@ -106,7 +106,16 @@ private:
std::unique_ptr<MediaFilter> filter_;
std::mutex mtx_;
- void setRotation(int rotation);
+ void sendFrameDirect(const std::shared_ptr<jami::MediaFrame>&);
+ void sendFrameTransformed(AVFrame* frame);
+
+ /**
+ * Apply required transformations before sending frames to clients/observers:
+ * - Transfer the frame from gpu to main memory, if needed.
+ * - Rotate the frame as needed.
+ * - Apply cropping as needed
+ */
+ std::shared_ptr<VideoFrame> applyTransform(VideoFrame& frame);
#ifdef DEBUG_FPS
unsigned frameCount_;
diff --git a/src/media/video/video_scaler.cpp b/src/media/video/video_scaler.cpp
index 11e751ec3b6553a48baf2d4f6de1b28e2b7ae53f..daeac3144163b2673ddfd064125aa06932ce178a 100644
--- a/src/media/video/video_scaler.cpp
+++ b/src/media/video/video_scaler.cpp
@@ -42,14 +42,12 @@ VideoScaler::~VideoScaler()
void
VideoScaler::scale(const VideoFrame& input, VideoFrame& output){
- scale(input, output.pointer());
+ scale(input.pointer(), output.pointer());
}
void
-VideoScaler::scale(const VideoFrame& input, AVFrame* output_frame)
+VideoScaler::scale(const AVFrame* input_frame, AVFrame* output_frame)
{
- const auto input_frame = input.pointer();
-
ctx_ = sws_getCachedContext(ctx_,
input_frame->width,
input_frame->height,
diff --git a/src/media/video/video_scaler.h b/src/media/video/video_scaler.h
index 50707c71acda47018afdb4ef26f2e5634c6dd679..907eb9ce3e586b6552e220fa8a4649a4782ed7f9 100644
--- a/src/media/video/video_scaler.h
+++ b/src/media/video/video_scaler.h
@@ -39,7 +39,7 @@ public:
VideoScaler();
~VideoScaler();
void reset();
- void scale(const VideoFrame& input, AVFrame* output);
+ void scale(const AVFrame* input, AVFrame* output);
void scale(const VideoFrame& input, VideoFrame& output);
void scale_with_aspect(const VideoFrame& input, VideoFrame& output);
void scale_and_pad(const VideoFrame& input,
diff --git a/src/observer.h b/src/observer.h
index 121b3ecd5e3f658cfbc0a9ca599f2f6a452a571f..03619f0516aba8463ac01356cc69ad9002186e5c 100644
--- a/src/observer.h
+++ b/src/observer.h
@@ -111,10 +111,10 @@ public:
return false;
}
- int getObserversCount()
+ size_t getObserversCount()
{
std::lock_guard<std::mutex> lk(mutex_);
- return observers_.size();
+ return observers_.size() + priority_observers_.size();
}
protected: