diff --git a/src/media/media_decoder.cpp b/src/media/media_decoder.cpp
index 40e2f05b410d14d7fa522535fe5140257ec182c4..b3d63d6b44449d70ebaafba6a53feea3eccd081f 100644
--- a/src/media/media_decoder.cpp
+++ b/src/media/media_decoder.cpp
@@ -227,9 +227,10 @@ MediaDecoder::setupStream(AVMediaType mediaType)
 #ifdef RING_ACCEL
     if (mediaType == AVMEDIA_TYPE_VIDEO) {
         if (enableAccel_) {
-            accel_ = video::HardwareAccel::setupDecoder(decoderCtx_->codec_id);
+            accel_ = video::HardwareAccel::setupDecoder(decoderCtx_->codec_id,
+                decoderCtx_->width, decoderCtx_->height);
             if (accel_) {
-                accel_->setDetails(decoderCtx_, &options_);
+                accel_->setDetails(decoderCtx_);
                 decoderCtx_->opaque = accel_.get();
             }
         } else if (Manager::instance().videoPreferences.getDecodingAccelerated()) {
diff --git a/src/media/media_encoder.cpp b/src/media/media_encoder.cpp
index 5991909b1c7ef9b7eb2adb4e81faff01fc60ba4b..7d3a9be176577371a4bb92ee5c52b5bb197b4c5b 100644
--- a/src/media/media_encoder.cpp
+++ b/src/media/media_encoder.cpp
@@ -162,7 +162,7 @@ MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo)
 {
     if (systemCodecInfo.mediaType == MEDIA_AUDIO) {
         audioCodec_ = systemCodecInfo.name;
-        return initStream(systemCodecInfo);
+        return initStream(systemCodecInfo, nullptr);
     } else {
         videoCodec_ = systemCodecInfo.name;
         // TODO only support 1 audio stream and 1 video stream per encoder
@@ -174,17 +174,17 @@ MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo)
 }
 
 int
-MediaEncoder::initStream(const std::string& codecName)
+MediaEncoder::initStream(const std::string& codecName, AVBufferRef* framesCtx)
 {
     const auto codecInfo = getSystemCodecContainer()->searchCodecByName(codecName, MEDIA_ALL);
     if (codecInfo)
-        return initStream(*codecInfo);
+        return initStream(*codecInfo, framesCtx);
     else
         return -1;
 }
 
 int
-MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo)
+MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo, AVBufferRef* framesCtx)
 {
     AVCodec* outputCodec = nullptr;
     AVCodecContext* encoderCtx = nullptr;
@@ -192,7 +192,8 @@ MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo)
     if (systemCodecInfo.mediaType == MEDIA_VIDEO) {
         if (enableAccel_) {
             if (accel_ = video::HardwareAccel::setupEncoder(
-                static_cast<AVCodecID>(systemCodecInfo.avcodecId), videoOpts_.width, videoOpts_.height)) {
+                static_cast<AVCodecID>(systemCodecInfo.avcodecId),
+                videoOpts_.width, videoOpts_.height, framesCtx)) {
                 outputCodec = avcodec_find_encoder_by_name(accel_->getCodecName().c_str());
             }
         } else {
@@ -221,7 +222,7 @@ MediaEncoder::initStream(const SystemCodecInfo& systemCodecInfo)
 
 #ifdef RING_ACCEL
     if (accel_) {
-        accel_->setDetails(encoderCtx, &options_);
+        accel_->setDetails(encoderCtx);
         encoderCtx->opaque = accel_.get();
     }
 #endif
@@ -383,28 +384,49 @@ MediaEncoder::encode(VideoFrame& input, bool is_keyframe,
                      int64_t frame_number)
 {
     if (!initialized_) {
-        initStream(videoCodec_);
+        initStream(videoCodec_, input.pointer()->hw_frames_ctx);
         startIO();
     }
 
-    /* Prepare a frame suitable to our encoder frame format,
-     * keeping also the input aspect ratio.
-     */
-    libav_utils::fillWithBlack(scaledFrame_.pointer());
-
-    scaler_.scale_with_aspect(input, scaledFrame_);
-
-    // Copy frame so the VideoScaler can still use the software frame (input)
-    VideoFrame copy;
-    copy.copyFrom(scaledFrame_);
+    AVFrame* frame;
+#ifdef RING_ACCEL
+    auto desc = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(input.format()));
+    bool isHardware = desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL);
+    std::unique_ptr<VideoFrame> framePtr;
+    if (accel_ && accel_->isLinked()) {
+        // Fully accelerated pipeline, skip main memory
+        frame = input.pointer();
+    } else if (isHardware) {
+        // Hardware decoded frame, transfer back to main memory
+        // Transfer to GPU if we have a hardware encoder
+        AVPixelFormat pix = (accel_ ? accel_->getSoftwareFormat() : AV_PIX_FMT_YUV420P);
+        framePtr = video::HardwareAccel::transferToMainMemory(input, pix);
+        if (accel_)
+            framePtr = accel_->transfer(*framePtr);
+        frame = framePtr->pointer();
+    } else if (accel_) {
+        // Software decoded frame with a hardware encoder, convert to accepted format first
+        auto pix = accel_->getSoftwareFormat();
+        if (input.format() != pix) {
+            framePtr = scaler_.convertFormat(input, pix);
+            framePtr = accel_->transfer(*framePtr);
+        } else {
+            framePtr = accel_->transfer(input);
+        }
+        frame = framePtr->pointer();
+    } else {
+#endif
+        libav_utils::fillWithBlack(scaledFrame_.pointer());
+        scaler_.scale_with_aspect(input, scaledFrame_);
+        frame = scaledFrame_.pointer();
+#ifdef RING_ACCEL
+    }
+#endif
 
-    auto frame = copy.pointer();
     AVCodecContext* enc = encoders_[currentStreamIdx_];
-    // ideally, time base is the inverse of framerate, but this may not always be the case
-    if (enc->framerate.num == enc->time_base.den && enc->framerate.den == enc->time_base.num)
-        frame->pts = frame_number;
-    else
-        frame->pts = (frame_number / (rational<int64_t>(enc->framerate) * rational<int64_t>(enc->time_base))).real<int64_t>();
+    frame->pts = frame_number;
+    if (enc->framerate.num != enc->time_base.den || enc->framerate.den != enc->time_base.num)
+        frame->pts /= (rational<int64_t>(enc->framerate) * rational<int64_t>(enc->time_base)).real<int64_t>();
 
     if (is_keyframe) {
         frame->pict_type = AV_PICTURE_TYPE_I;
@@ -414,19 +436,6 @@ MediaEncoder::encode(VideoFrame& input, bool is_keyframe,
         frame->key_frame = 0;
     }
 
-#ifdef RING_ACCEL
-    // NOTE needs to be at same scope as call to encode
-    std::unique_ptr<VideoFrame> framePtr;
-    if (accel_) {
-        framePtr = accel_->transfer(copy);
-        if (!framePtr) {
-            RING_ERR() << "Hardware encoding failure";
-            return -1;
-        }
-        frame = framePtr->pointer();
-    }
-#endif
-
     return encode(frame, currentStreamIdx_);
 }
 #endif // RING_VIDEO
@@ -454,7 +463,7 @@ MediaEncoder::encode(AVFrame* frame, int streamIdx)
         // Initialize on first video frame, or first audio frame if no video stream
         bool isVideo = (frame->width > 0 && frame->height > 0);
         if (isVideo or not videoOpts_.isValid()) {
-            initStream(videoCodec_);
+            initStream(videoCodec_, frame->hw_frames_ctx);
             startIO();
         } else {
             return 0;
diff --git a/src/media/media_encoder.h b/src/media/media_encoder.h
index 0c253543225d9ebe280c53b98a227aa51a665101..aca43124149b21adad2ba5d044ebcf9277b80fd6 100644
--- a/src/media/media_encoder.h
+++ b/src/media/media_encoder.h
@@ -110,8 +110,8 @@ private:
     AVCodecContext* prepareEncoderContext(AVCodec* outputCodec, bool is_video);
     void forcePresetX264(AVCodecContext* encoderCtx);
     void extractProfileLevelID(const std::string &parameters, AVCodecContext *ctx);
-    int initStream(const std::string& codecName);
-    int initStream(const SystemCodecInfo& systemCodecInfo);
+    int initStream(const std::string& codecName, AVBufferRef* framesCtx);
+    int initStream(const SystemCodecInfo& systemCodecInfo, AVBufferRef* framesCtx);
     void openIOContext();
     void startIO();
 
diff --git a/src/media/video/accel.cpp b/src/media/video/accel.cpp
index 1e7e7216c7bc38dbe3ba0d72d5d70b2f8fff4400..fd4e7ab320e25218ce145d1e1b74f2e033e4d920 100644
--- a/src/media/video/accel.cpp
+++ b/src/media/video/accel.cpp
@@ -104,7 +104,7 @@ HardwareAccel::transfer(const VideoFrame& frame)
             return nullptr;
         }
 
-        return transferToMainMemory(frame, AV_PIX_FMT_NV12);
+        return transferToMainMemory(frame, swFormat_);
     } else if (type_ == CODEC_ENCODER) {
         auto input = frame.pointer();
         if (input->format != swFormat_) {
@@ -141,14 +141,14 @@ HardwareAccel::transfer(const VideoFrame& frame)
 }
 
 void
-HardwareAccel::setDetails(AVCodecContext* codecCtx, AVDictionary** /*d*/)
+HardwareAccel::setDetails(AVCodecContext* codecCtx)
 {
     if (type_ == CODEC_DECODER) {
         codecCtx->hw_device_ctx = av_buffer_ref(deviceCtx_);
         codecCtx->get_format = getFormatCb;
         codecCtx->thread_safe_callbacks = 1;
     } else if (type_ == CODEC_ENCODER) {
-        codecCtx->hw_device_ctx = av_buffer_ref(deviceCtx_);
+        // encoder doesn't need a device context, only a frame context
         codecCtx->hw_frames_ctx = av_buffer_ref(framesCtx_);
     }
 }
@@ -204,6 +204,28 @@ HardwareAccel::initFrame(int width, int height)
     return ret >= 0;
 }
 
+bool
+HardwareAccel::linkHardware(AVBufferRef* framesCtx)
+{
+    if (framesCtx) {
+        // Force sw_format to match swFormat_. Frame is never transferred to main
+        // memory when hardware is linked, so the sw_format doesn't matter.
+        auto hw = reinterpret_cast<AVHWFramesContext*>(framesCtx->data);
+        hw->sw_format = swFormat_;
+
+        if (framesCtx_)
+            av_buffer_unref(&framesCtx_);
+        framesCtx_ = av_buffer_ref(framesCtx);
+        if ((linked_ = (framesCtx_ != nullptr))) {
+            RING_DBG() << "Hardware transcoding pipeline successfully set up for"
+                << " encoder '" << getCodecName() << "'";
+        }
+        return linked_;
+    } else {
+        return false;
+    }
+}
+
 std::unique_ptr<VideoFrame>
 HardwareAccel::transferToMainMemory(const VideoFrame& frame, AVPixelFormat desiredFormat)
 {
@@ -232,7 +254,7 @@ HardwareAccel::transferToMainMemory(const VideoFrame& frame, AVPixelFormat desir
 }
 
 std::unique_ptr<HardwareAccel>
-HardwareAccel::setupDecoder(AVCodecID id)
+HardwareAccel::setupDecoder(AVCodecID id, int width, int height)
 {
     static const HardwareAPI apiList[] = {
         { "vaapi", AV_PIX_FMT_VAAPI, AV_PIX_FMT_NV12, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_VP8, AV_CODEC_ID_MJPEG } },
@@ -243,7 +265,7 @@ HardwareAccel::setupDecoder(AVCodecID id)
     for (const auto& api : apiList) {
         if (std::find(api.supportedCodecs.begin(), api.supportedCodecs.end(), id) != api.supportedCodecs.end()) {
             auto accel = std::make_unique<HardwareAccel>(id, api.name, api.format, api.swFormat, CODEC_DECODER);
-            if (accel->initDevice()) {
+            if (accel->initDevice() && accel->initFrame(width, height)) {
                 RING_DBG() << "Attempting to use hardware decoder " << accel->getCodecName() << " with " << api.name;
                 return accel;
             }
@@ -254,7 +276,7 @@ HardwareAccel::setupDecoder(AVCodecID id)
 }
 
 std::unique_ptr<HardwareAccel>
-HardwareAccel::setupEncoder(AVCodecID id, int width, int height)
+HardwareAccel::setupEncoder(AVCodecID id, int width, int height, AVBufferRef* framesCtx)
 {
     static const HardwareAPI apiList[] = {
         { "vaapi", AV_PIX_FMT_VAAPI, AV_PIX_FMT_NV12, { AV_CODEC_ID_H264, AV_CODEC_ID_MJPEG, AV_CODEC_ID_VP8 } },
@@ -267,7 +289,9 @@ HardwareAccel::setupEncoder(AVCodecID id, int width, int height)
             auto accel = std::make_unique<HardwareAccel>(id, api.name, api.format, api.swFormat, CODEC_ENCODER);
             const auto& codecName = accel->getCodecName();
             if (avcodec_find_encoder_by_name(codecName.c_str())) {
-                if (accel->initDevice() && accel->initFrame(width, height)) {
+                // Set up a fully accelerated pipeline, else fallback to using the main memory
+                if (accel->linkHardware(framesCtx)
+                    || (accel->initDevice() && accel->initFrame(width, height))) {
                     RING_DBG() << "Attempting to use hardware encoder " << codecName;
                     return accel;
                 }
diff --git a/src/media/video/accel.h b/src/media/video/accel.h
index 85fdbe88c620ee37bf6e8b7e60f280512c4609b5..b6f1a7f7c9f498538483f268be4c03ac47f85672 100644
--- a/src/media/video/accel.h
+++ b/src/media/video/accel.h
@@ -37,12 +37,13 @@ public:
     /**
      * Static factory method for hardware decoding.
      */
-    static std::unique_ptr<HardwareAccel> setupDecoder(AVCodecID id);
+    static std::unique_ptr<HardwareAccel> setupDecoder(AVCodecID id, int width, int height);
 
     /**
      * Static factory method for hardware encoding.
      */
-    static std::unique_ptr<HardwareAccel> setupEncoder(AVCodecID id, int width, int height);
+    static std::unique_ptr<HardwareAccel> setupEncoder(AVCodecID id, int width, int height,
+        AVBufferRef* framesCtx = nullptr);
 
     /**
      * Transfers a hardware decoded frame back to main memory. Should be called after
@@ -86,19 +87,27 @@ public:
 
     /**
      * Gets the name of the codec.
-     * Decoding: equivalent to avcodec_get_name(id_)
+     * Decoding: avcodec_get_name(id_)
      * Encoding: avcodec_get_name(id_) + '_' + name_
      */
     std::string getCodecName() const;
 
+    /**
+     * Returns whether or not the decoder is linked to an encoder or vice-versa. Being linked
+     * means an encoder can directly use the decoder's hardware frame, without first
+     * transferring it to main memory.
+     */
+    bool isLinked() const { return linked_; }
+
     /**
      * Set some extra details in the codec context. Should be called after a successful
      * setup (setupDecoder or setupEncoder).
-     * For decoding, sets the hw_device_ctx and get_format callback. For encoding, sets
-     * hw_device_ctx and hw_frames_ctx, and may set some hardware specific options in
-     * the dictionary.
+     * For decoding, sets the hw_device_ctx and get_format callback. If the decoder has
+     * a frames context, mark as linked.
+     * For encoding, sets hw_device_ctx and hw_frames_ctx, and may set some hardware
+     * codec options.
      */
-    void setDetails(AVCodecContext* codecCtx, AVDictionary** d);
+    void setDetails(AVCodecContext* codecCtx);
 
     /**
      * Transfers a hardware decoded frame back to main memory. Should be called after
@@ -110,6 +119,12 @@ public:
      */
     std::unique_ptr<VideoFrame> transfer(const VideoFrame& frame);
 
+    /**
+     * Links this HardwareAccel's frames context with the passed in context. This serves
+     * to skip transferring a decoded frame back to main memory before encoding.
+     */
+    bool linkHardware(AVBufferRef* framesCtx);
+
 private:
     bool initDevice();
     bool initFrame(int width, int height);
@@ -119,6 +134,7 @@ private:
     AVPixelFormat format_ {AV_PIX_FMT_NONE};
     AVPixelFormat swFormat_ {AV_PIX_FMT_NONE};
     CodecType type_ {CODEC_NONE};
+    bool linked_ {false};
 
     AVBufferRef* deviceCtx_ {nullptr};
     AVBufferRef* framesCtx_ {nullptr};
diff --git a/src/media/video/video_sender.cpp b/src/media/video/video_sender.cpp
index 205b24c1fc869546065f46c71fab477fcdba5d51..ca47036c9b927133ee7492dfc81a8c3c056dd724 100644
--- a/src/media/video/video_sender.cpp
+++ b/src/media/video/video_sender.cpp
@@ -106,13 +106,7 @@ VideoSender::encodeAndSendVideo(VideoFrame& input_frame)
                 changeOrientationCallback_(rotation_);
         }
 
-#ifdef RING_ACCEL
-        auto framePtr = HardwareAccel::transferToMainMemory(input_frame, AV_PIX_FMT_NV12);
-        auto& swFrame = *framePtr;
-#else
-        auto& swFrame = input_frame;
-#endif
-        if (videoEncoder_->encode(swFrame, is_keyframe, frameNumber_++) < 0)
+        if (videoEncoder_->encode(input_frame, is_keyframe, frameNumber_++) < 0)
             RING_ERR("encoding failed");
     }
 #ifdef DEBUG_SDP