Make WebrtcVideoEncoder asynchronous.
Previously WebrtcVideoEncoder interface was synchronous, so it was
incompatible with HW encoders. Refactored it to make it asynchronous.
Change-Id: I1b0f3ce6dda657ecddb4ac84cf8132a22b473308
Reviewed-on: https://chromium-review.googlesource.com/541122
Reviewed-by: Scott Nichols <[email protected]>
Commit-Queue: Sergey Ulanov <[email protected]>
Cr-Commit-Position: refs/heads/master@{#481297}diff --git a/remoting/codec/BUILD.gn b/remoting/codec/BUILD.gn
index 85896de..1eee958 100644
--- a/remoting/codec/BUILD.gn
+++ b/remoting/codec/BUILD.gn
@@ -15,6 +15,8 @@
"video_encoder_verbatim.h",
"video_encoder_vpx.cc",
"video_encoder_vpx.h",
+ "webrtc_video_encoder_proxy.cc",
+ "webrtc_video_encoder_proxy.h",
"webrtc_video_encoder_vpx.cc",
"webrtc_video_encoder_vpx.h",
]
diff --git a/remoting/codec/webrtc_video_encoder.h b/remoting/codec/webrtc_video_encoder.h
index bf097c2..09d53e2 100644
--- a/remoting/codec/webrtc_video_encoder.h
+++ b/remoting/codec/webrtc_video_encoder.h
@@ -51,16 +51,21 @@
int quantizer;
};
+ typedef base::OnceCallback<void(std::unique_ptr<EncodedFrame>)>
+ EncodeCallback;
+
virtual ~WebrtcVideoEncoder() {}
// Encode an image stored in |frame|. If frame.updated_region() is empty
// then the encoder may return a frame (e.g. to top-off previously-encoded
// portions of the frame to higher quality) or return nullptr to indicate that
- // there is no work to do. |frame| may be nullptr. This case must be handled
- // the same as if frame.updated_region() is empty.
- virtual std::unique_ptr<EncodedFrame> Encode(
- const webrtc::DesktopFrame* frame,
- const FrameParams& param) = 0;
+ // there is no work to do. |frame| may be nullptr, which is equivalent to a
+ // frame with an empty updated_region(). |done| callback may be called
+ // synchronously. It must not be called if the encoder is destroyed while
+ // request is pending.
+ virtual void Encode(std::unique_ptr<webrtc::DesktopFrame> frame,
+ const FrameParams& param,
+ EncodeCallback done) = 0;
};
} // namespace remoting
diff --git a/remoting/codec/webrtc_video_encoder_proxy.cc b/remoting/codec/webrtc_video_encoder_proxy.cc
new file mode 100644
index 0000000..4ef47cc
--- /dev/null
+++ b/remoting/codec/webrtc_video_encoder_proxy.cc
@@ -0,0 +1,69 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "remoting/codec/webrtc_video_encoder_proxy.h"
+
+#include "base/bind.h"
+#include "base/threading/sequenced_task_runner_handle.h"
+#include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
+
+namespace remoting {
+
+class WebrtcVideoEncoderProxy::Core {
+ public:
+ Core(std::unique_ptr<WebrtcVideoEncoder> encoder)
+ : encoder_(std::move(encoder)),
+ main_task_runner_(base::SequencedTaskRunnerHandle::Get()) {}
+ ~Core() {}
+
+ void Encode(std::unique_ptr<webrtc::DesktopFrame> frame,
+ const FrameParams& params,
+ EncodeCallback done) {
+ encoder_->Encode(std::move(frame), params,
+ base::BindOnce(&Core::OnEncoded, base::Unretained(this),
+ std::move(done)));
+ }
+
+ private:
+ void OnEncoded(EncodeCallback done,
+ std::unique_ptr<WebrtcVideoEncoder::EncodedFrame> frame) {
+ main_task_runner_->PostTask(
+ FROM_HERE, base::BindOnce(std::move(done), std::move(frame)));
+ }
+
+ std::unique_ptr<WebrtcVideoEncoder> encoder_;
+ scoped_refptr<base::SequencedTaskRunner> main_task_runner_;
+};
+
+WebrtcVideoEncoderProxy::WebrtcVideoEncoderProxy(
+ std::unique_ptr<WebrtcVideoEncoder> encoder,
+ scoped_refptr<base::SequencedTaskRunner> encode_task_runner)
+ : core_(new Core(std::move(encoder))),
+ encode_task_runner_(encode_task_runner),
+ weak_factory_(this) {}
+
+WebrtcVideoEncoderProxy::~WebrtcVideoEncoderProxy() {
+ encode_task_runner_->DeleteSoon(FROM_HERE, core_.release());
+}
+
+void WebrtcVideoEncoderProxy::Encode(
+ std::unique_ptr<webrtc::DesktopFrame> frame,
+ const FrameParams& params,
+ EncodeCallback done) {
+ encode_task_runner_->PostTask(
+ FROM_HERE,
+ base::BindOnce(
+ &Core::Encode, base::Unretained(core_.get()), std::move(frame),
+ params,
+ base::BindOnce(&WebrtcVideoEncoderProxy::OnEncoded,
+ weak_factory_.GetWeakPtr(), std::move(done))));
+}
+
+void WebrtcVideoEncoderProxy::OnEncoded(
+ EncodeCallback done,
+ std::unique_ptr<WebrtcVideoEncoder::EncodedFrame> frame) {
+ std::move(done).Run(std::move(frame));
+}
+
+} // namespace remoting
diff --git a/remoting/codec/webrtc_video_encoder_proxy.h b/remoting/codec/webrtc_video_encoder_proxy.h
new file mode 100644
index 0000000..bcd261f
--- /dev/null
+++ b/remoting/codec/webrtc_video_encoder_proxy.h
@@ -0,0 +1,46 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef REMOTING_CODEC_WEBRTC_VIDEO_ENCODER_PROXY_H_
+#define REMOTING_CODEC_WEBRTC_VIDEO_ENCODER_PROXY_H_
+
+#include <memory>
+
+#include "base/memory/ref_counted.h"
+#include "base/memory/weak_ptr.h"
+#include "remoting/codec/webrtc_video_encoder.h"
+
+namespace base {
+class SequencedTaskRunner;
+} // namespace base
+
+namespace remoting {
+
+// WebrtcVideoEncoder implementation that runs encoder on a background thread.
+class WebrtcVideoEncoderProxy : public WebrtcVideoEncoder {
+ public:
+ WebrtcVideoEncoderProxy(
+ std::unique_ptr<WebrtcVideoEncoder> encoder,
+ scoped_refptr<base::SequencedTaskRunner> encode_task_runner);
+ ~WebrtcVideoEncoderProxy() override;
+
+ // WebrtcVideoEncoder interface.
+ void Encode(std::unique_ptr<webrtc::DesktopFrame> frame,
+ const FrameParams& params,
+ EncodeCallback done) override;
+
+ private:
+ class Core;
+
+ void OnEncoded(EncodeCallback done,
+ std::unique_ptr<WebrtcVideoEncoder::EncodedFrame> frame);
+
+ std::unique_ptr<Core> core_;
+ scoped_refptr<base::SequencedTaskRunner> encode_task_runner_;
+ base::WeakPtrFactory<WebrtcVideoEncoderProxy> weak_factory_;
+};
+
+} // namespace remoting
+
+#endif // REMOTING_CODEC_WEBRTC_VIDEO_ENCODER_PROXY_H_
diff --git a/remoting/codec/webrtc_video_encoder_vpx.cc b/remoting/codec/webrtc_video_encoder_vpx.cc
index da198feb..28b2bf1 100644
--- a/remoting/codec/webrtc_video_encoder_vpx.cc
+++ b/remoting/codec/webrtc_video_encoder_vpx.cc
@@ -7,6 +7,7 @@
#include <utility>
#include "base/bind.h"
+#include "base/callback.h"
#include "base/logging.h"
#include "base/memory/ptr_util.h"
#include "base/sys_info.h"
@@ -273,9 +274,9 @@
}
}
-std::unique_ptr<WebrtcVideoEncoder::EncodedFrame> WebrtcVideoEncoderVpx::Encode(
- const webrtc::DesktopFrame* frame,
- const FrameParams& params) {
+void WebrtcVideoEncoderVpx::Encode(std::unique_ptr<webrtc::DesktopFrame> frame,
+ const FrameParams& params,
+ EncodeCallback done) {
webrtc::DesktopSize previous_frame_size =
image_ ? webrtc::DesktopSize(image_->w, image_->h)
: webrtc::DesktopSize();
@@ -284,7 +285,8 @@
// Don't need to send anything until we get the first non-null frame.
if (frame_size.is_empty()) {
- return nullptr;
+ std::move(done).Run(nullptr);
+ return;
}
DCHECK_GE(frame_size.width(), 32);
@@ -304,7 +306,7 @@
webrtc::DesktopRegion updated_region;
// Convert the updated capture data ready for encode.
- PrepareImage(frame, &updated_region);
+ PrepareImage(frame.get(), &updated_region);
// Update active map based on updated region.
if (params.clear_active_map)
@@ -373,7 +375,7 @@
}
}
- return encoded_frame;
+ std::move(done).Run(std::move(encoded_frame));
}
WebrtcVideoEncoderVpx::WebrtcVideoEncoderVpx(bool use_vp9)
diff --git a/remoting/codec/webrtc_video_encoder_vpx.h b/remoting/codec/webrtc_video_encoder_vpx.h
index 5c5cfd4..a657497 100644
--- a/remoting/codec/webrtc_video_encoder_vpx.h
+++ b/remoting/codec/webrtc_video_encoder_vpx.h
@@ -41,8 +41,9 @@
void SetLosslessColor(bool want_lossless);
// WebrtcVideoEncoder interface.
- std::unique_ptr<EncodedFrame> Encode(const webrtc::DesktopFrame* frame,
- const FrameParams& params) override;
+ void Encode(std::unique_ptr<webrtc::DesktopFrame> frame,
+ const FrameParams& params,
+ EncodeCallback done) override;
private:
explicit WebrtcVideoEncoderVpx(bool use_vp9);
diff --git a/remoting/protocol/webrtc_video_stream.cc b/remoting/protocol/webrtc_video_stream.cc
index 0167d3a..ac8ffdf 100644
--- a/remoting/protocol/webrtc_video_stream.cc
+++ b/remoting/protocol/webrtc_video_stream.cc
@@ -7,10 +7,9 @@
#include <utility>
#include "base/logging.h"
-#include "base/single_thread_task_runner.h"
-#include "base/task_runner_util.h"
-#include "base/threading/thread_task_runner_handle.h"
+#include "base/memory/ptr_util.h"
#include "remoting/base/constants.h"
+#include "remoting/codec/webrtc_video_encoder_proxy.h"
#include "remoting/codec/webrtc_video_encoder_vpx.h"
#include "remoting/protocol/frame_stats.h"
#include "remoting/protocol/host_video_stats_dispatcher.h"
@@ -42,11 +41,6 @@
uint32_t capturer_id = 0;
};
-struct WebrtcVideoStream::EncodedFrameWithStats {
- std::unique_ptr<WebrtcVideoEncoder::EncodedFrame> frame;
- std::unique_ptr<FrameStats> stats;
-};
-
WebrtcVideoStream::WebrtcVideoStream()
: video_stats_dispatcher_(kStreamLabel), weak_factory_(this) {}
@@ -58,13 +52,12 @@
}
peer_connection_->RemoveStream(stream_.get());
}
- encode_task_runner_->DeleteSoon(FROM_HERE, encoder_.release());
}
void WebrtcVideoStream::Start(
std::unique_ptr<webrtc::DesktopCapturer> desktop_capturer,
WebrtcTransport* webrtc_transport,
- scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner) {
+ scoped_refptr<base::SequencedTaskRunner> encode_task_runner) {
DCHECK(thread_checker_.CalledOnValidThread());
DCHECK(webrtc_transport);
DCHECK(desktop_capturer);
@@ -147,8 +140,8 @@
std::unique_ptr<webrtc::DesktopFrame> frame) {
DCHECK(thread_checker_.CalledOnValidThread());
- captured_frame_stats_->capture_ended_time = base::TimeTicks::Now();
- captured_frame_stats_->capture_delay =
+ current_frame_stats_->capture_ended_time = base::TimeTicks::Now();
+ current_frame_stats_->capture_delay =
base::TimeDelta::FromMilliseconds(frame ? frame->capture_time_ms() : 0);
WebrtcVideoEncoder::FrameParams frame_params;
@@ -168,17 +161,15 @@
if (observer_)
observer_->OnVideoSizeChanged(this, frame_size_, frame_dpi_);
}
+
+ current_frame_stats_->capturer_id = frame->capturer_id();
}
DCHECK(encoder_);
-
- base::PostTaskAndReplyWithResult(
- encode_task_runner_.get(), FROM_HERE,
- base::Bind(&WebrtcVideoStream::EncodeFrame, encoder_.get(),
- base::Passed(std::move(frame)), frame_params,
- base::Passed(std::move(captured_frame_stats_))),
- base::Bind(&WebrtcVideoStream::OnFrameEncoded,
- weak_factory_.GetWeakPtr()));
+ current_frame_stats_->encode_started_time = base::TimeTicks::Now();
+ encoder_->Encode(
+ std::move(frame), frame_params,
+ base::Bind(&WebrtcVideoStream::OnFrameEncoded, base::Unretained(this)));
}
void WebrtcVideoStream::OnChannelInitialized(
@@ -194,44 +185,30 @@
void WebrtcVideoStream::CaptureNextFrame() {
DCHECK(thread_checker_.CalledOnValidThread());
- captured_frame_stats_.reset(new FrameStats());
- captured_frame_stats_->capture_started_time = base::TimeTicks::Now();
- captured_frame_stats_->input_event_timestamps =
+ current_frame_stats_.reset(new FrameStats());
+ current_frame_stats_->capture_started_time = base::TimeTicks::Now();
+ current_frame_stats_->input_event_timestamps =
event_timestamps_source_->TakeLastEventTimestamps();
capturer_->CaptureFrame();
}
-// static
-WebrtcVideoStream::EncodedFrameWithStats WebrtcVideoStream::EncodeFrame(
- WebrtcVideoEncoder* encoder,
- std::unique_ptr<webrtc::DesktopFrame> frame,
- WebrtcVideoEncoder::FrameParams params,
- std::unique_ptr<WebrtcVideoStream::FrameStats> stats) {
- EncodedFrameWithStats result;
- result.stats = std::move(stats);
- result.stats->encode_started_time = base::TimeTicks::Now();
- result.frame = encoder->Encode(frame.get(), params);
- result.stats->encode_ended_time = base::TimeTicks::Now();
- if (frame) {
- result.stats->capturer_id = frame->capturer_id();
- }
- return result;
-}
-
-void WebrtcVideoStream::OnFrameEncoded(EncodedFrameWithStats frame) {
+void WebrtcVideoStream::OnFrameEncoded(
+ std::unique_ptr<WebrtcVideoEncoder::EncodedFrame> frame) {
DCHECK(thread_checker_.CalledOnValidThread());
- HostFrameStats stats;
- scheduler_->OnFrameEncoded(frame.frame.get(), &stats);
+ current_frame_stats_->encode_ended_time = base::TimeTicks::Now();
- if (!frame.frame) {
+ HostFrameStats stats;
+ scheduler_->OnFrameEncoded(frame.get(), &stats);
+
+ if (!frame) {
return;
}
webrtc::EncodedImageCallback::Result result =
webrtc_transport_->video_encoder_factory()->SendEncodedFrame(
- *frame.frame, frame.stats->capture_started_time);
+ *frame, current_frame_stats_->capture_started_time);
if (result.error != webrtc::EncodedImageCallback::Result::OK) {
// TODO(sergeyu): Stop the stream.
LOG(ERROR) << "Failed to send video frame.";
@@ -240,30 +217,31 @@
// Send FrameStats message.
if (video_stats_dispatcher_.is_connected()) {
- stats.frame_size = frame.frame ? frame.frame->data.size() : 0;
+ stats.frame_size = frame ? frame->data.size() : 0;
- if (!frame.stats->input_event_timestamps.is_null()) {
+ if (!current_frame_stats_->input_event_timestamps.is_null()) {
stats.capture_pending_delay =
- frame.stats->capture_started_time -
- frame.stats->input_event_timestamps.host_timestamp;
+ current_frame_stats_->capture_started_time -
+ current_frame_stats_->input_event_timestamps.host_timestamp;
stats.latest_event_timestamp =
- frame.stats->input_event_timestamps.client_timestamp;
+ current_frame_stats_->input_event_timestamps.client_timestamp;
}
- stats.capture_delay = frame.stats->capture_delay;
+ stats.capture_delay = current_frame_stats_->capture_delay;
// Total overhead time for IPC and threading when capturing frames.
stats.capture_overhead_delay =
- (frame.stats->capture_ended_time - frame.stats->capture_started_time) -
+ (current_frame_stats_->capture_ended_time -
+ current_frame_stats_->capture_started_time) -
stats.capture_delay;
- stats.encode_pending_delay =
- frame.stats->encode_started_time - frame.stats->capture_ended_time;
+ stats.encode_pending_delay = current_frame_stats_->encode_started_time -
+ current_frame_stats_->capture_ended_time;
- stats.encode_delay =
- frame.stats->encode_ended_time - frame.stats->encode_started_time;
+ stats.encode_delay = current_frame_stats_->encode_ended_time -
+ current_frame_stats_->encode_started_time;
- stats.capturer_id = frame.stats->capturer_id;
+ stats.capturer_id = current_frame_stats_->capturer_id;
video_stats_dispatcher_.OnVideoFrameStats(result.frame_id, stats);
}
@@ -272,9 +250,11 @@
void WebrtcVideoStream::OnEncoderCreated(webrtc::VideoCodecType codec_type) {
DCHECK(thread_checker_.CalledOnValidThread());
if (codec_type == webrtc::kVideoCodecVP8) {
- encoder_ = WebrtcVideoEncoderVpx::CreateForVP8();
+ encoder_ = base::MakeUnique<WebrtcVideoEncoderProxy>(
+ WebrtcVideoEncoderVpx::CreateForVP8(), encode_task_runner_);
} else if (codec_type == webrtc::kVideoCodecVP9) {
- encoder_ = WebrtcVideoEncoderVpx::CreateForVP9();
+ encoder_ = base::MakeUnique<WebrtcVideoEncoderProxy>(
+ WebrtcVideoEncoderVpx::CreateForVP9(), encode_task_runner_);
} else {
LOG(FATAL) << "Unknown codec type: " << codec_type;
}
diff --git a/remoting/protocol/webrtc_video_stream.h b/remoting/protocol/webrtc_video_stream.h
index 8b7998ca..64578b48 100644
--- a/remoting/protocol/webrtc_video_stream.h
+++ b/remoting/protocol/webrtc_video_stream.h
@@ -42,7 +42,7 @@
void Start(std::unique_ptr<webrtc::DesktopCapturer> desktop_capturer,
WebrtcTransport* webrtc_transport,
- scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner);
+ scoped_refptr<base::SequencedTaskRunner> encode_task_runner);
// VideoStream interface.
void SetEventTimestampsSource(scoped_refptr<InputEventTimestampsSource>
@@ -54,7 +54,6 @@
private:
struct FrameStats;
- struct EncodedFrameWithStats;
// webrtc::DesktopCapturer::Callback interface.
void OnCaptureResult(webrtc::DesktopCapturer::Result result,
@@ -67,13 +66,7 @@
// Called by the |scheduler_|.
void CaptureNextFrame();
- // Task running on the encoder thread to encode the |frame|.
- static EncodedFrameWithStats EncodeFrame(
- WebrtcVideoEncoder* encoder,
- std::unique_ptr<webrtc::DesktopFrame> frame,
- WebrtcVideoEncoder::FrameParams params,
- std::unique_ptr<WebrtcVideoStream::FrameStats> stats);
- void OnFrameEncoded(EncodedFrameWithStats frame);
+ void OnFrameEncoded(std::unique_ptr<WebrtcVideoEncoder::EncodedFrame> frame);
void OnEncoderCreated(webrtc::VideoCodecType codec_type);
@@ -82,7 +75,7 @@
// Used to send across encoded frames.
WebrtcTransport* webrtc_transport_ = nullptr;
// Task runner used to run |encoder_|.
- scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner_;
+ scoped_refptr<base::SequencedTaskRunner> encode_task_runner_;
// Used to encode captured frames. Always accessed on the encode thread.
std::unique_ptr<WebrtcVideoEncoder> encoder_;
@@ -94,7 +87,7 @@
HostVideoStatsDispatcher video_stats_dispatcher_;
// Stats of the frame that's being captured.
- std::unique_ptr<FrameStats> captured_frame_stats_;
+ std::unique_ptr<FrameStats> current_frame_stats_;
std::unique_ptr<WebrtcFrameScheduler> scheduler_;
