From cc8fb024626b54948230706175890c933bd06896 Mon Sep 17 00:00:00 2001
From: agsantos <aline.gondimsantos@savoirfairelinux.com>
Date: Tue, 16 Nov 2021 09:23:04 -0500
Subject: [PATCH] x11: window sharing
+ If in screen sharing, the hardware acceleration encoding is disabled
GitLab: https://git.jami.net/savoirfairelinux/jami-project/-/issues/1294
Change-Id: I24982b4454d724bb86059a50a3c2d01fe6fab147
---
contrib/src/ffmpeg/rules.mak | 3 +
.../src/ffmpeg/screen-sharing-x11-fix.patch | 302 ++++++++++++++++++
src/media/media_decoder.cpp | 65 +++-
src/media/media_decoder.h | 16 +-
src/media/media_device.h | 2 +
src/media/media_encoder.cpp | 72 +++--
src/media/media_encoder.h | 1 +
src/media/video/sinkclient.cpp | 5 +-
src/media/video/video_device_monitor.cpp | 4 +-
src/media/video/video_input.cpp | 68 ++--
src/media/video/video_input.h | 6 +-
src/media/video/video_rtp_session.cpp | 10 +-
src/media/video/video_sender.cpp | 21 +-
src/media/video/video_sender.h | 3 +-
14 files changed, 511 insertions(+), 67 deletions(-)
create mode 100644 contrib/src/ffmpeg/screen-sharing-x11-fix.patch
diff --git a/contrib/src/ffmpeg/rules.mak b/contrib/src/ffmpeg/rules.mak
index df6b3711e5..2a9a58391a 100644
--- a/contrib/src/ffmpeg/rules.mak
+++ b/contrib/src/ffmpeg/rules.mak
@@ -3,9 +3,11 @@ FFMPEG_URL := https://git.ffmpeg.org/gitweb/ffmpeg.git/snapshot/$(FFMPEG_HASH).t
PKGS+=ffmpeg
+ifndef HAVE_LINUX
ifeq ($(call need_pkg,"libavutil >= 55.75.100 libavcodec >= 57.106.101 libavformat >= 57.82.100 libavdevice >= 57.8.101 libavfilter >= 6.105.100 libswscale >= 4.7.103 libswresample >= 2.9.100"),)
PKGS_FOUND += ffmpeg
endif
+endif
DEPS_ffmpeg = iconv zlib vpx opus speex x264
@@ -348,6 +350,7 @@ ffmpeg: ffmpeg-$(FFMPEG_HASH).tar.gz
$(APPLY) $(SRC)/ffmpeg/rtp_ext_abs_send_time.patch
$(APPLY) $(SRC)/ffmpeg/libopusdec-enable-FEC.patch
$(APPLY) $(SRC)/ffmpeg/libopusenc-enable-FEC.patch
+ $(APPLY) $(SRC)/ffmpeg/screen-sharing-x11-fix.patch
ifdef HAVE_IOS
$(APPLY) $(SRC)/ffmpeg/ios-disable-b-frames.patch
endif
diff --git a/contrib/src/ffmpeg/screen-sharing-x11-fix.patch b/contrib/src/ffmpeg/screen-sharing-x11-fix.patch
new file mode 100644
index 0000000000..9ae7dafebe
--- /dev/null
+++ b/contrib/src/ffmpeg/screen-sharing-x11-fix.patch
@@ -0,0 +1,302 @@
+From f498c487a24bd39f1adfaa51b415e93456d36612 Mon Sep 17 00:00:00 2001
+From: agsantos <aline.gondimsantos@savoirfairelinux.com>
+Date: Wed, 17 Nov 2021 12:37:32 -0500
+Subject: [PATCH] Screen sharing x11 fixes
+
++ We can now have a single stream in the x11grab, which can be updated to follow window resizing
++ Due to stream reinit, shm may cause memory issues and was removed
++ Adds one option (is_area) that defines if we are grabing a region of the display/window or the hole screen/window.
+
+note: This is a custom patch for later rebase
+---
+ libavdevice/xcbgrab.c | 186 ++++++++++--------------------------------
+ 1 file changed, 45 insertions(+), 141 deletions(-)
+
+diff --git a/libavdevice/xcbgrab.c b/libavdevice/xcbgrab.c
+index 8e3292e577..406716682f 100644
+--- a/libavdevice/xcbgrab.c
++++ b/libavdevice/xcbgrab.c
+@@ -29,11 +29,6 @@
+ #include <xcb/xfixes.h>
+ #endif
+
+-#if CONFIG_LIBXCB_SHM
+-#include <sys/shm.h>
+-#include <xcb/shm.h>
+-#endif
+-
+ #if CONFIG_LIBXCB_SHAPE
+ #include <xcb/shape.h>
+ #endif
+@@ -53,9 +48,6 @@ typedef struct XCBGrabContext {
+ xcb_connection_t *conn;
+ xcb_screen_t *screen;
+ xcb_window_t window;
+-#if CONFIG_LIBXCB_SHM
+- AVBufferPool *shm_pool;
+-#endif
+ int64_t time_frame;
+ AVRational time_base;
+ int64_t frame_duration;
+@@ -72,10 +64,9 @@ typedef struct XCBGrabContext {
+ int region_border;
+ int centered;
+ int select_region;
++ int is_area;
+
+ const char *framerate;
+-
+- int has_shm;
+ } XCBGrabContext;
+
+ #define FOLLOW_CENTER -1
+@@ -97,6 +88,7 @@ static const AVOption options[] = {
+ { "show_region", "Show the grabbing region.", OFFSET(show_region), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, D },
+ { "region_border", "Set the region border thickness.", OFFSET(region_border), AV_OPT_TYPE_INT, { .i64 = 3 }, 1, 128, D },
+ { "select_region", "Select the grabbing region graphically using the pointer.", OFFSET(select_region), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, D },
++ { "is_area", "Define if we are grabing a region of the display/window.", OFFSET(is_area), AV_OPT_TYPE_INT, { .i64 = 1 }, 0, 1, D },
+ { NULL },
+ };
+
+@@ -216,99 +208,6 @@ static int64_t wait_frame(AVFormatContext *s, AVPacket *pkt)
+ return curtime;
+ }
+
+-#if CONFIG_LIBXCB_SHM
+-static int check_shm(xcb_connection_t *conn)
+-{
+- xcb_shm_query_version_cookie_t cookie = xcb_shm_query_version(conn);
+- xcb_shm_query_version_reply_t *reply;
+-
+- reply = xcb_shm_query_version_reply(conn, cookie, NULL);
+- if (reply) {
+- free(reply);
+- return 1;
+- }
+-
+- return 0;
+-}
+-
+-static void free_shm_buffer(void *opaque, uint8_t *data)
+-{
+- shmdt(data);
+-}
+-
+-static AVBufferRef *allocate_shm_buffer(void *opaque, buffer_size_t size)
+-{
+- xcb_connection_t *conn = opaque;
+- xcb_shm_seg_t segment;
+- AVBufferRef *ref;
+- uint8_t *data;
+- int id;
+-
+- id = shmget(IPC_PRIVATE, size, IPC_CREAT | 0777);
+- if (id == -1)
+- return NULL;
+-
+- segment = xcb_generate_id(conn);
+- xcb_shm_attach(conn, segment, id, 0);
+- data = shmat(id, NULL, 0);
+- shmctl(id, IPC_RMID, 0);
+- if ((intptr_t)data == -1 || !data)
+- return NULL;
+-
+- ref = av_buffer_create(data, size, free_shm_buffer, (void *)(ptrdiff_t)segment, 0);
+- if (!ref)
+- shmdt(data);
+-
+- return ref;
+-}
+-
+-static int xcbgrab_frame_shm(AVFormatContext *s, AVPacket *pkt)
+-{
+- XCBGrabContext *c = s->priv_data;
+- xcb_shm_get_image_cookie_t iq;
+- xcb_shm_get_image_reply_t *img;
+- xcb_drawable_t drawable = c->window_id;
+- xcb_generic_error_t *e = NULL;
+- AVBufferRef *buf;
+- xcb_shm_seg_t segment;
+-
+- buf = av_buffer_pool_get(c->shm_pool);
+- if (!buf) {
+- av_log(s, AV_LOG_ERROR, "Could not get shared memory buffer.\n");
+- return AVERROR(ENOMEM);
+- }
+- segment = (xcb_shm_seg_t)av_buffer_pool_buffer_get_opaque(buf);
+-
+- iq = xcb_shm_get_image(c->conn, drawable,
+- c->x, c->y, c->width, c->height, ~0,
+- XCB_IMAGE_FORMAT_Z_PIXMAP, segment, 0);
+- img = xcb_shm_get_image_reply(c->conn, iq, &e);
+-
+- xcb_flush(c->conn);
+-
+- if (e) {
+- av_log(s, AV_LOG_ERROR,
+- "Cannot get the image data "
+- "event_error: response_type:%u error_code:%u "
+- "sequence:%u resource_id:%u minor_code:%u major_code:%u.\n",
+- e->response_type, e->error_code,
+- e->sequence, e->resource_id, e->minor_code, e->major_code);
+-
+- free(e);
+- av_buffer_unref(&buf);
+- return AVERROR(EACCES);
+- }
+-
+- free(img);
+-
+- pkt->buf = buf;
+- pkt->data = buf->data;
+- pkt->size = c->frame_size;
+-
+- return 0;
+-}
+-#endif /* CONFIG_LIBXCB_SHM */
+-
+ #if CONFIG_LIBXCB_XFIXES
+ static int check_xfixes(xcb_connection_t *conn)
+ {
+@@ -462,14 +361,7 @@ static int xcbgrab_read_packet(AVFormatContext *s, AVPacket *pkt)
+ if (c->show_region)
+ xcbgrab_update_region(s, win_x, win_y);
+
+-#if CONFIG_LIBXCB_SHM
+- if (c->has_shm && xcbgrab_frame_shm(s, pkt) < 0) {
+- av_log(s, AV_LOG_WARNING, "Continuing without shared memory.\n");
+- c->has_shm = 0;
+- }
+-#endif
+- if (!c->has_shm)
+- ret = xcbgrab_frame(s, pkt);
++ ret = xcbgrab_frame(s, pkt);
+ pkt->dts = pkt->pts = pts;
+ pkt->duration = c->frame_duration;
+
+@@ -488,11 +380,8 @@ static av_cold int xcbgrab_read_close(AVFormatContext *s)
+ {
+ XCBGrabContext *ctx = s->priv_data;
+
+-#if CONFIG_LIBXCB_SHM
+- av_buffer_pool_uninit(&ctx->shm_pool);
+-#endif
+-
+ xcb_disconnect(ctx->conn);
++ ctx->conn = NULL;
+
+ return 0;
+ }
+@@ -572,7 +461,15 @@ static int pixfmt_from_pixmap_format(AVFormatContext *s, int depth,
+ static int create_stream(AVFormatContext *s)
+ {
+ XCBGrabContext *c = s->priv_data;
+- AVStream *st = avformat_new_stream(s, NULL);
++
++ // If we try to open another stream to x11grab, there is no reason
++ // to keep more than one stream in the context.
++ AVStream *st;
++ if (!s->nb_streams) {
++ st = avformat_new_stream(s, NULL);
++ } else {
++ st = s->streams[0];
++ }
+ xcb_get_geometry_cookie_t gc;
+ xcb_get_geometry_reply_t *geo;
+ int64_t frame_size_bits;
+@@ -594,11 +491,26 @@ static int create_stream(AVFormatContext *s)
+ return AVERROR_EXTERNAL;
+ }
+
++ // av_log(s, AV_LOG_ERROR, "Capture is_area %d\n", c->is_area);
++ // Width and Height are not 0 only when we set a window area to share
++ // This if may be valid only in the first call to create_stream
+ if (!c->width || !c->height) {
++ // av_log(s, AV_LOG_ERROR, "Capture area!\n");
++ c->is_area = 0;
++ c->width = geo->width;
++ c->height = geo->height;
++ }
++ // If not a predefined area, then we should follow geometry changes
++ // This can be valid only on the second call onwards
++ if (!c->is_area && (c->width != geo->width || c->height != geo->height)) {
+ c->width = geo->width;
+ c->height = geo->height;
+ }
+
++ // av_log(s, AV_LOG_ERROR, "Capture area %dx%d at position %d.%d\n",
++ // c->width, c->height,
++ // c->x, c->y);
++
+ if (c->x + c->width > geo->width ||
+ c->y + c->height > geo->height) {
+ av_log(s, AV_LOG_ERROR,
+@@ -628,13 +540,6 @@ static int create_stream(AVFormatContext *s)
+ }
+ c->frame_size = frame_size_bits / 8;
+
+-#if CONFIG_LIBXCB_SHM
+- c->shm_pool = av_buffer_pool_init2(c->frame_size + AV_INPUT_BUFFER_PADDING_SIZE,
+- c->conn, allocate_shm_buffer, NULL);
+- if (!c->shm_pool)
+- return AVERROR(ENOMEM);
+-#endif
+-
+ st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
+ st->codecpar->codec_id = AV_CODEC_ID_RAWVIDEO;
+ st->codecpar->width = c->width;
+@@ -829,23 +734,26 @@ static av_cold int xcbgrab_read_header(AVFormatContext *s)
+ sscanf(s->url, "+%d,%d", &c->x, &c->y);
+ }
+
+- c->conn = xcb_connect(display_name[0] ? display_name : NULL, &screen_num);
+- av_freep(&display_name);
++ if (!c->conn || !c->screen) {
++ xcbgrab_read_close(s);
++ c->conn = xcb_connect(display_name[0] ? display_name : NULL, &screen_num);
++ av_freep(&display_name);
+
+- if ((ret = xcb_connection_has_error(c->conn))) {
+- av_log(s, AV_LOG_ERROR, "Cannot open display %s, error %d.\n",
+- s->url[0] ? s->url : "default", ret);
+- return AVERROR(EIO);
+- }
++ if ((ret = xcb_connection_has_error(c->conn))) {
++ av_log(s, AV_LOG_ERROR, "Cannot open display %s, error %d.\n",
++ s->url[0] ? s->url : "default", ret);
++ return AVERROR(EIO);
++ }
+
+- setup = xcb_get_setup(c->conn);
++ setup = xcb_get_setup(c->conn);
+
+- c->screen = get_screen(setup, screen_num);
+- if (!c->screen) {
+- av_log(s, AV_LOG_ERROR, "The screen %d does not exist.\n",
+- screen_num);
+- xcbgrab_read_close(s);
+- return AVERROR(EIO);
++ c->screen = get_screen(setup, screen_num);
++ if (!c->screen) {
++ av_log(s, AV_LOG_ERROR, "The screen %d does not exist.\n",
++ screen_num);
++ xcbgrab_read_close(s);
++ return AVERROR(EIO);
++ }
+ }
+
+ if (c->window_id == XCB_NONE)
+@@ -876,10 +784,6 @@ static av_cold int xcbgrab_read_header(AVFormatContext *s)
+ return ret;
+ }
+
+-#if CONFIG_LIBXCB_SHM
+- c->has_shm = check_shm(c->conn);
+-#endif
+-
+ #if CONFIG_LIBXCB_XFIXES
+ if (c->draw_mouse) {
+ if (!(c->draw_mouse = check_xfixes(c->conn))) {
+--
+2.30.2
+
diff --git a/src/media/media_decoder.cpp b/src/media/media_decoder.cpp
index f7c1c44730..5aed8616dc 100644
--- a/src/media/media_decoder.cpp
+++ b/src/media/media_decoder.cpp
@@ -117,6 +117,10 @@ MediaDemuxer::openInput(const DeviceParams& params)
if (!params.pixel_format.empty()) {
av_dict_set(&options_, "pixel_format", params.pixel_format.c_str(), 0);
}
+ if (!params.window_id.empty()) {
+ av_dict_set(&options_, "window_id", params.window_id.c_str(), 0);
+ }
+ av_dict_set(&options_, "is_area", std::to_string(params.is_area).c_str(), 0);
#if defined(__APPLE__) && TARGET_OS_MAC
std::string input = params.name;
@@ -142,7 +146,12 @@ MediaDemuxer::openInput(const DeviceParams& params)
if (ret) {
JAMI_ERR("avformat_open_input failed: %s", libav_utils::getError(ret).c_str());
} else {
- JAMI_DBG("Using format %s", params.format.c_str());
+ baseWidth_ = inputCtx_->streams[0]->codecpar->width;
+ baseHeight_ = inputCtx_->streams[0]->codecpar->height;
+ JAMI_DBG("Using format %s and resolution %dx%d",
+ params.format.c_str(),
+ baseWidth_,
+ baseHeight_);
}
return ret;
@@ -321,18 +330,35 @@ MediaDemuxer::decode()
av_packet_free(
&p);
});
+ if (inputParams_.format == "x11grab") {
+ auto ret = inputCtx_->iformat->read_header(inputCtx_);
+ if (ret == AVERROR_EXTERNAL) {
+ JAMI_ERR("Couldn't read frame: %s\n", libav_utils::getError(ret).c_str());
+ return Status::ReadError;
+ }
+ auto codecpar = inputCtx_->streams[0]->codecpar;
+ if (baseHeight_ != codecpar->height || baseWidth_ != codecpar->width) {
+ baseHeight_ = codecpar->height;
+ baseWidth_ = codecpar->width;
+ inputParams_.height = ((baseHeight_ >> 3) << 3);
+ inputParams_.width = ((baseWidth_ >> 3) << 3);
+ return Status::RestartRequired;
+ }
+ }
int ret = av_read_frame(inputCtx_, packet.get());
if (ret == AVERROR(EAGAIN)) {
/*no data available. Calculate time until next frame.
- We do not use the emulated frame mechanism from the decoder because it will affect all platforms.
- With the current implementation, the demuxer will be waiting just in case when av_read_frame
- returns EAGAIN. For some platforms, av_read_frame is blocking and it will never happen.
+ We do not use the emulated frame mechanism from the decoder because it will affect all
+ platforms. With the current implementation, the demuxer will be waiting just in case when
+ av_read_frame returns EAGAIN. For some platforms, av_read_frame is blocking and it will
+ never happen.
*/
if (inputParams_.framerate.numerator() == 0)
return Status::Success;
- rational<double> frameTime = 1e6/inputParams_.framerate;
- int64_t timeToSleep = lastReadPacketTime_ - av_gettime_relative() + frameTime.real<int64_t>();
+ rational<double> frameTime = 1e6 / inputParams_.framerate;
+ int64_t timeToSleep = lastReadPacketTime_ - av_gettime_relative()
+ + frameTime.real<int64_t>();
if (timeToSleep <= 0) {
return Status::Success;
}
@@ -340,6 +366,8 @@ MediaDemuxer::decode()
return Status::Success;
} else if (ret == AVERROR_EOF) {
return Status::EndOfFile;
+ } else if (ret == AVERROR(EACCES)) {
+ return Status::RestartRequired;
} else if (ret < 0) {
JAMI_ERR("Couldn't read frame: %s\n", libav_utils::getError(ret).c_str());
return Status::ReadError;
@@ -570,9 +598,11 @@ MediaDecoder::decode(AVPacket& packet)
avcodec_flush_buffers(decoderCtx_);
setupStream();
return DecodeStatus::FallBack;
- } else
+ }
#endif
- return ret == AVERROR_EOF ? DecodeStatus::Success : DecodeStatus::DecodeError;
+ avcodec_flush_buffers(decoderCtx_);
+ setupStream();
+ return ret == AVERROR_EOF ? DecodeStatus::Success : DecodeStatus::DecodeError;
}
auto f = (inputDecoder_->type == AVMEDIA_TYPE_VIDEO)
@@ -583,10 +613,10 @@ MediaDecoder::decode(AVPacket& packet)
if (resolutionChangedCallback_) {
if (decoderCtx_->width != width_ or decoderCtx_->height != height_) {
JAMI_DBG("Resolution changed from %dx%d to %dx%d",
- width_,
- height_,
- decoderCtx_->width,
- decoderCtx_->height);
+ width_,
+ height_,
+ decoderCtx_->width,
+ decoderCtx_->height);
width_ = decoderCtx_->width;
height_ = decoderCtx_->height;
resolutionChangedCallback_(width_, height_);
@@ -613,7 +643,8 @@ MediaDecoder::decode(AVPacket& packet)
lastTimestamp_ = frame->pts;
if (emulateRate_ and packetTimestamp != AV_NOPTS_VALUE) {
auto startTime = avStream_->start_time == AV_NOPTS_VALUE ? 0 : avStream_->start_time;
- rational<double> frame_time = rational<double>(getTimeBase()) * (packetTimestamp - startTime);
+ rational<double> frame_time = rational<double>(getTimeBase())
+ * (packetTimestamp - startTime);
auto target_relative = static_cast<std::int64_t>(frame_time.real() * 1e6);
auto target_absolute = startTime_ + target_relative;
if (target_relative < seekTime_) {
@@ -645,7 +676,13 @@ MediaDecoder::setSeekTime(int64_t time)
MediaDemuxer::Status
MediaDecoder::decode()
{
- return demuxer_->decode();
+ auto ret = demuxer_->decode();
+ if (ret == MediaDemuxer::Status::RestartRequired) {
+ avcodec_flush_buffers(decoderCtx_);
+ setupStream();
+ ret = MediaDemuxer::Status::EndOfFile;
+ }
+ return ret;
}
#ifdef ENABLE_VIDEO
diff --git a/src/media/media_decoder.h b/src/media/media_decoder.h
index a296d5397b..4d47882a09 100644
--- a/src/media/media_decoder.h
+++ b/src/media/media_decoder.h
@@ -85,7 +85,14 @@ public:
MediaDemuxer();
~MediaDemuxer();
- enum class Status { Success, EndOfFile, ReadBufferOverflow, ReadError, FallBack };
+ enum class Status {
+ Success,
+ EndOfFile,
+ ReadBufferOverflow,
+ ReadError,
+ FallBack,
+ RestartRequired
+ };
enum class CurrentState { Demuxing, Finished };
using StreamCallback = std::function<DecodeStatus(AVPacket&)>;
@@ -145,6 +152,8 @@ private:
void pushFrameFrom(std::queue<std::unique_ptr<AVPacket, std::function<void(AVPacket*)>>>& buffer,
bool isAudio,
std::mutex& mutex);
+ int baseWidth_ {};
+ int baseHeight_ {};
};
class MediaDecoder
@@ -190,7 +199,10 @@ public:
MediaStream getStream(std::string name = "") const;
- void setResolutionChangedCallback(std::function<void(int, int)> cb) { resolutionChangedCallback_ = std::move(cb); }
+ void setResolutionChangedCallback(std::function<void(int, int)> cb)
+ {
+ resolutionChangedCallback_ = std::move(cb);
+ }
void setFEC(bool enable) { fecEnabled_ = enable; }
diff --git a/src/media/media_device.h b/src/media/media_device.h
index 3556a9e0db..1a0a6bf79b 100644
--- a/src/media/media_device.h
+++ b/src/media/media_device.h
@@ -51,6 +51,8 @@ struct DeviceParams
int offset_x {};
int offset_y {};
int orientation {};
+ std::string window_id {};
+ int is_area {};
};
} // namespace jami
diff --git a/src/media/media_encoder.cpp b/src/media/media_encoder.cpp
index 805108eb68..02798a96c2 100644
--- a/src/media/media_encoder.cpp
+++ b/src/media/media_encoder.cpp
@@ -101,12 +101,13 @@ MediaEncoder::setOptions(const MediaStream& opts)
// Make sure width and height are even (required by x264)
// This is especially for image/gif streaming, as video files and cameras usually have even
// resolutions
- videoOpts_.width -= videoOpts_.width % 2;
- videoOpts_.height -= videoOpts_.height % 2;
+ videoOpts_.width = ((videoOpts_.width >> 3) << 3);
+ videoOpts_.height = ((videoOpts_.height >> 3) << 3);
if (!videoOpts_.frameRate)
videoOpts_.frameRate = 30;
- if (!videoOpts_.bitrate)
- videoOpts_.bitrate = SystemCodecInfo::DEFAULT_VIDEO_BITRATE;;
+ if (!videoOpts_.bitrate) {
+ videoOpts_.bitrate = SystemCodecInfo::DEFAULT_VIDEO_BITRATE;
+ }
} else {
audioOpts_ = opts;
}
@@ -211,8 +212,15 @@ MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo, AVBufferRef* fr
else if (systemCodecInfo.mediaType == MEDIA_AUDIO)
mediaType = AVMEDIA_TYPE_AUDIO;
+ AVStream* stream;
// add video stream to outputformat context
- AVStream* stream = avformat_new_stream(outputCtx_, outputCodec_);
+ if (outputCtx_->nb_streams <= 0)
+ stream = avformat_new_stream(outputCtx_, outputCodec_);
+ else {
+ stream = outputCtx_->streams[0];
+ stream->codecpar->width = videoOpts_.width;
+ stream->codecpar->height = videoOpts_.height;
+ }
if (!stream)
throw MediaEncoderException("Could not allocate stream");
@@ -363,7 +371,9 @@ MediaEncoder::startIO()
#ifdef ENABLE_VIDEO
int
-MediaEncoder::encode(const std::shared_ptr<VideoFrame>& input, bool is_keyframe, int64_t frame_number)
+MediaEncoder::encode(const std::shared_ptr<VideoFrame>& input,
+ bool is_keyframe,
+ int64_t frame_number)
{
if (!initialized_) {
initStream(videoCodec_, input->pointer()->hw_frames_ctx);
@@ -390,7 +400,7 @@ MediaEncoder::encode(const std::shared_ptr<VideoFrame>& input, bool is_keyframe,
avframe->pts = frame_number;
if (enc->framerate.num != enc->time_base.den || enc->framerate.den != enc->time_base.num)
avframe->pts /= (rational<int64_t>(enc->framerate) * rational<int64_t>(enc->time_base))
- .real<int64_t>();
+ .real<int64_t>();
if (is_keyframe) {
avframe->pict_type = AV_PICTURE_TYPE_I;
@@ -858,8 +868,7 @@ MediaEncoder::initH264(AVCodecContext* encoderCtx, uint64_t br)
uint64_t maxBitrate = 1000 * br;
// 200 Kbit/s -> CRF40
// 6 Mbit/s -> CRF23
- uint8_t crf = (uint8_t) std::round(
- LOGREG_PARAM_A + LOGREG_PARAM_B*log(maxBitrate));
+ uint8_t crf = (uint8_t) std::round(LOGREG_PARAM_A + LOGREG_PARAM_B * log(maxBitrate));
// bufsize parameter impact the variation of the bitrate, reduce to half the maxrate to limit
// peak and congestion
// https://trac.ffmpeg.org/wiki/Limiting%20the%20output%20bitrate
@@ -895,8 +904,8 @@ MediaEncoder::initH265(AVCodecContext* encoderCtx, uint64_t br)
// CRF) https://slhck.info/video/2017/02/24/crf-guide.html
// 200 Kbit/s -> CRF35
// 6 Mbit/s -> CRF18
- uint8_t crf = (uint8_t) std::round(
- LOGREG_PARAM_A_HEVC + LOGREG_PARAM_B_HEVC*log(maxBitrate));
+ uint8_t crf = (uint8_t) std::round(LOGREG_PARAM_A_HEVC
+ + LOGREG_PARAM_B_HEVC * log(maxBitrate));
uint64_t bufSize = maxBitrate / 2;
av_opt_set_int(encoderCtx, "crf", crf, AV_OPT_SEARCH_CHILDREN);
av_opt_set_int(encoderCtx, "maxrate", maxBitrate, AV_OPT_SEARCH_CHILDREN);
@@ -938,8 +947,7 @@ MediaEncoder::initVP8(AVCodecContext* encoderCtx, uint64_t br)
uint64_t maxBitrate = 1000 * br;
// 200 Kbit/s -> CRF40
// 6 Mbit/s -> CRF23
- uint8_t crf = (uint8_t) std::round(
- LOGREG_PARAM_A + LOGREG_PARAM_B*log(maxBitrate));
+ uint8_t crf = (uint8_t) std::round(LOGREG_PARAM_A + LOGREG_PARAM_B * log(maxBitrate));
uint64_t bufSize = maxBitrate / 2;
av_opt_set(encoderCtx, "quality", "realtime", AV_OPT_SEARCH_CHILDREN);
@@ -1194,11 +1202,12 @@ MediaEncoder::testH265Accel()
}
int
-MediaEncoder::getHWFrame(const std::shared_ptr<VideoFrame>& input, std::shared_ptr<VideoFrame>& output)
+MediaEncoder::getHWFrame(const std::shared_ptr<VideoFrame>& input,
+ std::shared_ptr<VideoFrame>& output)
{
try {
#if defined(TARGET_OS_IOS) && TARGET_OS_IOS
-// iOS
+ // iOS
if (accel_) {
auto pix = accel_->getSoftwareFormat();
if (input->format() != pix) {
@@ -1211,7 +1220,7 @@ MediaEncoder::getHWFrame(const std::shared_ptr<VideoFrame>& input, std::shared_p
output = getScaledSWFrame(*input.get());
}
#elif !defined(__APPLE__)
-// Other Platforms
+ // Other Platforms
auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input->format()));
bool isHardware = desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL);
if (accel_ && accel_->isLinked() && isHardware) {
@@ -1229,7 +1238,7 @@ MediaEncoder::getHWFrame(const std::shared_ptr<VideoFrame>& input, std::shared_p
output = getScaledSWFrame(*input.get());
}
#else
-// macOS
+ // macOS
output = getScaledSWFrame(*input.get());
#endif
} catch (const std::runtime_error& e) {
@@ -1247,8 +1256,7 @@ MediaEncoder::getUnlinkedHWFrame(const VideoFrame& input)
std::shared_ptr<VideoFrame> framePtr = video::HardwareAccel::transferToMainMemory(input, pix);
if (!accel_) {
framePtr = scaler_.convertFormat(*framePtr, AV_PIX_FMT_YUV420P);
- }
- else {
+ } else {
framePtr = accel_->transfer(*framePtr);
}
return framePtr;
@@ -1276,4 +1284,30 @@ MediaEncoder::getScaledSWFrame(const VideoFrame& input)
return scaledFrame_;
}
+void
+MediaEncoder::resetStreams(int width, int height)
+{
+ // Only called by VideoSender!
+ initialized_ = false;
+ videoOpts_.width = width;
+ videoOpts_.height = height;
+
+ try {
+ flush();
+ if (outputCtx_) {
+ for (auto encoderCtx : encoders_) {
+ if (encoderCtx) {
+#ifndef _MSC_VER
+ avcodec_free_context(&encoderCtx);
+#else
+ avcodec_close(encoderCtx);
+#endif
+ }
+ }
+ encoders_.clear();
+ }
+ } catch (...) {
+ }
+}
+
} // namespace jami
diff --git a/src/media/media_encoder.h b/src/media/media_encoder.h
index 983ec23966..787f2eb9ed 100644
--- a/src/media/media_encoder.h
+++ b/src/media/media_encoder.h
@@ -77,6 +77,7 @@ public:
void setOptions(const MediaDescription& args);
int addStream(const SystemCodecInfo& codec);
void setIOContext(AVIOContext* ioctx) { ioCtx_ = ioctx; }
+ void resetStreams(int width, int height);
bool send(AVPacket& packet, int streamIdx = -1);
diff --git a/src/media/video/sinkclient.cpp b/src/media/video/sinkclient.cpp
index f9b9d1fd7a..3b6aa8fa18 100644
--- a/src/media/video/sinkclient.cpp
+++ b/src/media/video/sinkclient.cpp
@@ -350,6 +350,7 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
}
#endif
+ std::lock_guard<std::mutex> lock(mtx_);
if (avTarget_.push) {
auto outFrame = std::make_unique<VideoFrame>();
outFrame->copyFrom(*std::static_pointer_cast<VideoFrame>(frame_p));
@@ -363,8 +364,10 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
if (outFrame->height() != height_ || outFrame->width() != width_) {
setFrameSize(0, 0);
setFrameSize(outFrame->width(), outFrame->height());
+ return;
}
avTarget_.push(std::move(outFrame));
+ return;
}
bool doTransfer = (target_.pull != nullptr);
@@ -420,11 +423,11 @@ SinkClient::update(Observable<std::shared_ptr<MediaFrame>>* /*obs*/,
if (frame->height() != height_ || frame->width() != width_) {
setFrameSize(0, 0);
setFrameSize(frame->width(), frame->height());
+ return;
}
#if HAVE_SHM
shm_->renderFrame(*frame);
#endif
- std::lock_guard<std::mutex> lock(mtx_);
if (target_.pull) {
int width = frame->width();
int height = frame->height();
diff --git a/src/media/video/video_device_monitor.cpp b/src/media/video/video_device_monitor.cpp
index 8542bd69ed..9d2e4ecbfd 100644
--- a/src/media/video/video_device_monitor.cpp
+++ b/src/media/video/video_device_monitor.cpp
@@ -102,7 +102,7 @@ VideoDeviceMonitor::getDefaultDevice() const
{
std::lock_guard<std::mutex> l(lock_);
const auto it = findDeviceById(defaultDevice_);
- if (it == std::end(devices_))
+ if (it == std::end(devices_) || it->getDeviceId() == DEVICE_DESKTOP)
return {};
return it->getDeviceId();
}
@@ -112,7 +112,7 @@ VideoDeviceMonitor::getMRLForDefaultDevice() const
{
std::lock_guard<std::mutex> l(lock_);
const auto it = findDeviceById(defaultDevice_);
- if (it == std::end(devices_))
+ if (it == std::end(devices_) || it->getDeviceId() == DEVICE_DESKTOP)
return {};
static const std::string sep = DRing::Media::VideoProtocolPrefix::SEPARATOR;
return DRing::Media::VideoProtocolPrefix::CAMERA + sep + it->getDeviceId();
diff --git a/src/media/video/video_input.cpp b/src/media/video/video_input.cpp
index c14e2fc516..096473c10b 100644
--- a/src/media/video/video_input.cpp
+++ b/src/media/video/video_input.cpp
@@ -204,8 +204,7 @@ void
VideoInput::cleanup()
{
deleteDecoder(); // do it first to let a chance to last frame to be displayed
- detach(sink_.get());
- sink_->stop();
+ stopSink();
JAMI_DBG("VideoInput closed");
}
@@ -286,6 +285,10 @@ VideoInput::createDecoder()
[](void* data) -> int { return not static_cast<VideoInput*>(data)->isCapturing(); }, this);
bool ready = false, restartSink = false;
+ if (decOpts_.format == "x11grab" && !decOpts_.is_area) {
+ decOpts_.width = 0;
+ decOpts_.height = 0;
+ }
while (!ready && !isStopped_) {
// Retry to open the video till the input is opened
auto ret = decoder->openInput(decOpts_);
@@ -317,10 +320,14 @@ VideoInput::createDecoder()
return;
}
- decoder->decode(); // Populate AVCodecContext fields
+ auto ret = decoder->decode(); // Populate AVCodecContext fields
+ if (ret == MediaDemuxer::Status::ReadError) {
+ JAMI_INFO() << "Decoder error";
+ return;
+ }
- decOpts_.width = decoder->getWidth();
- decOpts_.height = decoder->getHeight();
+ decOpts_.width = ((decoder->getWidth() >> 3) << 3);
+ decOpts_.height = ((decoder->getHeight() >> 3) << 3);
decOpts_.framerate = decoder->getFps();
AVPixelFormat fmt = decoder->getPixelFormat();
if (fmt != AV_PIX_FMT_NONE) {
@@ -391,29 +398,48 @@ round2pow(unsigned i, unsigned n)
}
bool
-VideoInput::initX11(std::string display)
+VideoInput::initX11(const std::string& display)
{
+ // Patterns
+ // full screen sharing : :1+0,0 2560x1440 - SCREEN 1, POSITION 0X0, RESOLUTION 2560X1440
+ // area sharing : :1+882,211 1532x779 - SCREEN 1, POSITION 882x211, RESOLUTION 1532x779
+ // window sharing : :+1,0 0x0 window-id:0x0340021e - POSITION 0X0
size_t space = display.find(' ');
+ std::string windowIdStr = "window-id:";
+ size_t winIdPos = display.find(windowIdStr);
- clearOptions();
- decOpts_ = jami::getVideoDeviceMonitor().getDeviceParams(DEVICE_DESKTOP);
-
+ DeviceParams p = jami::getVideoDeviceMonitor().getDeviceParams(DEVICE_DESKTOP);
+ if (winIdPos != std::string::npos) {
+ p.window_id = display.substr(winIdPos + windowIdStr.size()); // "0x0340021e";
+ p.is_area = 0;
+ }
if (space != std::string::npos) {
- std::istringstream iss(display.substr(space + 1));
- char sep;
- unsigned w, h;
- iss >> w >> sep >> h;
- // round to 8 pixel block
- decOpts_.width = round2pow(w, 3);
- decOpts_.height = round2pow(h, 3);
- decOpts_.input = display.erase(space);
+ p.input = display.substr(1, space);
+ if (p.window_id.empty()) {
+ p.input = display.substr(0, space);
+ JAMI_INFO() << "p.window_id.empty()";
+ auto splits = jami::split_string_to_unsigned(display.substr(space + 1), 'x');
+ // round to 8 pixel block
+ p.width = round2pow(splits[0], 3);
+ p.height = round2pow(splits[1], 3);
+ p.is_area = 1;
+ }
} else {
- decOpts_.input = display;
- // decOpts_.video_size = "vga";
- decOpts_.width = default_grab_width;
- decOpts_.height = default_grab_height;
+ p.input = display;
+ p.width = default_grab_width;
+ p.height = default_grab_height;
+ p.is_area = 1;
}
+ auto dec = std::make_unique<MediaDecoder>();
+ if (dec->openInput(p) < 0 || dec->setupVideo() < 0)
+ return initCamera(jami::getVideoDeviceMonitor().getDefaultDevice());
+
+ clearOptions();
+ decOpts_ = p;
+ decOpts_.width = round2pow(dec->getStream().width, 3);
+ decOpts_.height = round2pow(dec->getStream().height, 3);
+
return true;
}
diff --git a/src/media/video/video_input.h b/src/media/video/video_input.h
index f63a997eb2..a72f433c58 100644
--- a/src/media/video/video_input.h
+++ b/src/media/video/video_input.h
@@ -99,7 +99,9 @@ public:
#endif
void setSuccessfulSetupCb(const std::function<void(MediaType, bool)>& cb)
- { onSuccessfulSetup_ = cb; }
+ {
+ onSuccessfulSetup_ = cb;
+ }
private:
NON_COPYABLE(VideoInput);
@@ -123,7 +125,7 @@ private:
// true if decOpts_ is ready to use, false if using promise/future
bool initCamera(const std::string& device);
- bool initX11(std::string display);
+ bool initX11(const std::string& display);
bool initAVFoundation(const std::string& display);
bool initFile(std::string path);
bool initGdiGrab(const std::string& params);
diff --git a/src/media/video/video_rtp_session.cpp b/src/media/video/video_rtp_session.cpp
index cd656a1a25..54bcd74a71 100644
--- a/src/media/video/video_rtp_session.cpp
+++ b/src/media/video/video_rtp_session.cpp
@@ -149,6 +149,7 @@ VideoRtpSession::startSender()
send_.linkableHW = conference_ == nullptr;
send_.bitrate = videoBitrateInfo_.videoBitrateCurrent;
+ bool isScreenScharing = localVideoParams_.format == "x11grab";
if (socketPair_)
initSeqVal_ = socketPair_->lastSeqValOut();
@@ -166,8 +167,13 @@ VideoRtpSession::startSender()
send_.bitrate,
static_cast<rational<int>>(localVideoParams_.framerate))
: videoMixer_->getStream("Video Sender");
- sender_.reset(
- new VideoSender(getRemoteRtpUri(), ms, send_, *socketPair_, initSeqVal_ + 1, mtu_));
+ sender_.reset(new VideoSender(getRemoteRtpUri(),
+ ms,
+ send_,
+ *socketPair_,
+ initSeqVal_ + 1,
+ mtu_,
+ isScreenScharing));
if (changeOrientationCallback_)
sender_->setChangeOrientationCallback(changeOrientationCallback_);
if (socketPair_)
diff --git a/src/media/video/video_sender.cpp b/src/media/video/video_sender.cpp
index 48a19d6382..fdd2c746c7 100644
--- a/src/media/video/video_sender.cpp
+++ b/src/media/video/video_sender.cpp
@@ -47,7 +47,8 @@ VideoSender::VideoSender(const std::string& dest,
const MediaDescription& args,
SocketPair& socketPair,
const uint16_t seqVal,
- uint16_t mtu)
+ uint16_t mtu,
+ bool isScreenScharing = false)
: muxContext_(socketPair.createIOContext(mtu))
, videoEncoder_(new MediaEncoder)
{
@@ -58,10 +59,12 @@ VideoSender::VideoSender(const std::string& dest,
videoEncoder_->addStream(args.codec->systemCodecInfo);
videoEncoder_->setInitSeqVal(seqVal);
videoEncoder_->setIOContext(muxContext_->getContext());
-
+#ifdef RING_ACCEL
+ if (isScreenScharing)
+ videoEncoder_->enableAccel(false);
+#endif
// Send local video codec in SmartInfo
Smartools::getInstance().setLocalVideoCodec(videoEncoder_->getVideoCodec());
-
// Send the resolution in smartInfo
Smartools::getInstance().setResolution("local", opts.width, opts.height);
}
@@ -74,6 +77,18 @@ VideoSender::~VideoSender()
void
VideoSender::encodeAndSendVideo(const std::shared_ptr<VideoFrame>& input_frame)
{
+ auto width = ((input_frame->width() >> 3) << 3);
+ auto height = ((input_frame->height() >> 3) << 3);
+ if (videoEncoder_->getWidth() != width || videoEncoder_->getHeight() != height) {
+ videoEncoder_->resetStreams(width, height);
+ forceKeyFrame();
+ // Send local video codec in SmartInfo
+ Smartools::getInstance().setLocalVideoCodec(videoEncoder_->getVideoCodec());
+ // Send the resolution in smartInfo
+ Smartools::getInstance().setResolution("local", width, height);
+ return;
+ }
+
int angle = input_frame->getOrientation();
if (rotation_ != angle) {
rotation_ = angle;
diff --git a/src/media/video/video_sender.h b/src/media/video/video_sender.h
index 005027576c..0d2ca3068c 100644
--- a/src/media/video/video_sender.h
+++ b/src/media/video/video_sender.h
@@ -48,7 +48,8 @@ public:
const MediaDescription& args,
SocketPair& socketPair,
const uint16_t seqVal,
- uint16_t mtu);
+ uint16_t mtu,
+ bool isScreenScharing);
~VideoSender();
--
GitLab