Wire up webrtc::RateAllocation to RtcVideoEncoder.
This will be necessary in order to properly use temporal and spatial
scalability in hardware accelerated video codecs.
Bug: chromium:794608
Change-Id: I029e044a33c95eb4bcc9db04b028d04a3cb7f922
Reviewed-on: https://chromium-review.googlesource.com/1016900
Commit-Queue: Erik Språng <[email protected]>
Reviewed-by: Kentaro Hara <[email protected]>
Reviewed-by: Dale Curtis <[email protected]>
Reviewed-by: Emircan Uysaler <[email protected]>
Cr-Commit-Position: refs/heads/master@{#560254}diff --git a/content/renderer/BUILD.gn b/content/renderer/BUILD.gn
index e1adb642..0fc9510b 100644
--- a/content/renderer/BUILD.gn
+++ b/content/renderer/BUILD.gn
@@ -743,8 +743,6 @@
"//third_party/webrtc/api:libjingle_peerconnection_api",
"//third_party/webrtc/api:optional",
"//third_party/webrtc/api:rtc_stats_api",
- "//third_party/webrtc/api/video:video_frame",
- "//third_party/webrtc/api/video:video_frame_i420",
"//third_party/webrtc/api/audio:aec3_factory",
"//third_party/webrtc/api/audio_codecs:audio_codecs_api",
"//third_party/webrtc/api/audio_codecs/L16:audio_decoder_L16",
@@ -757,6 +755,9 @@
"//third_party/webrtc/api/audio_codecs/isac:audio_encoder_isac",
"//third_party/webrtc/api/audio_codecs/opus:audio_decoder_opus",
"//third_party/webrtc/api/audio_codecs/opus:audio_encoder_opus",
+ "//third_party/webrtc/api/video:video_bitrate_allocation",
+ "//third_party/webrtc/api/video:video_frame",
+ "//third_party/webrtc/api/video:video_frame_i420",
"//third_party/webrtc/api/video_codecs:video_codecs_api",
"//third_party/webrtc/common_video:common_video",
"//third_party/webrtc/media:rtc_internal_video_codecs",
diff --git a/content/renderer/media/webrtc/rtc_video_encoder.cc b/content/renderer/media/webrtc/rtc_video_encoder.cc
index 02d58ce..bfa722c 100644
--- a/content/renderer/media/webrtc/rtc_video_encoder.cc
+++ b/content/renderer/media/webrtc/rtc_video_encoder.cc
@@ -28,6 +28,7 @@
#include "content/renderer/media/webrtc/webrtc_video_frame_adapter.h"
#include "media/base/bind_to_current_loop.h"
#include "media/base/bitstream_buffer.h"
+#include "media/base/video_bitrate_allocation.h"
#include "media/base/video_frame.h"
#include "media/base/video_util.h"
#include "media/video/gpu_video_accelerator_factories.h"
@@ -140,7 +141,8 @@
void UseOutputBitstreamBufferId(int32_t bitstream_buffer_id);
// Request encoding parameter change for the underlying encoder.
- void RequestEncodingParametersChange(uint32_t bitrate, uint32_t framerate);
+ void RequestEncodingParametersChange(webrtc::VideoBitrateAllocation bitrate,
+ uint32_t framerate);
void RegisterEncodeCompleteCallback(base::WaitableEvent* async_waiter,
int32_t* async_retval,
@@ -391,22 +393,41 @@
}
void RTCVideoEncoder::Impl::RequestEncodingParametersChange(
- uint32_t bitrate,
+ webrtc::VideoBitrateAllocation bitrate,
uint32_t framerate) {
- DVLOG(3) << __func__ << " bitrate=" << bitrate << ", framerate=" << framerate;
+ DVLOG(3) << __func__ << " bitrate=" << bitrate.ToString()
+ << ", framerate=" << framerate;
DCHECK(thread_checker_.CalledOnValidThread());
- // Check for overflow converting bitrate (kilobits/sec) to bits/sec.
- if (IsBitrateTooHigh(bitrate))
- return;
-
// This is a workaround to zero being temporarily provided, as part of the
// initial setup, by WebRTC.
- bitrate = std::max(1u, bitrate);
+ if (bitrate.get_sum_bps() == 0) {
+ bitrate.SetBitrate(0, 0, 1);
+ }
framerate = std::max(1u, framerate);
- if (video_encoder_)
- video_encoder_->RequestEncodingParametersChange(bitrate * 1000, framerate);
+ if (video_encoder_) {
+ media::VideoBitrateAllocation allocation;
+ for (size_t spatial_id = 0;
+ spatial_id < media::VideoBitrateAllocation::kMaxSpatialLayers;
+ ++spatial_id) {
+ for (size_t temporal_id = 0;
+ temporal_id < media::VideoBitrateAllocation::kMaxTemporalLayers;
+ ++temporal_id) {
+ // TODO(sprang): Clean this up if/when webrtc struct moves to int.
+ uint32_t layer_bitrate = bitrate.GetBitrate(spatial_id, temporal_id);
+ RTC_CHECK_LE(layer_bitrate,
+ static_cast<uint32_t>(std::numeric_limits<int>::max()));
+ if (!allocation.SetBitrate(spatial_id, temporal_id, layer_bitrate)) {
+ LOG(WARNING) << "Overflow in bitrate allocation: "
+ << bitrate.ToString();
+ break;
+ }
+ }
+ }
+ DCHECK_EQ(allocation.GetSumBps(), static_cast<int>(bitrate.get_sum_bps()));
+ video_encoder_->RequestEncodingParametersChange(allocation, framerate);
+ }
}
void RTCVideoEncoder::Impl::Destroy(base::WaitableEvent* async_waiter) {
@@ -922,8 +943,10 @@
return WEBRTC_VIDEO_CODEC_OK;
}
-int32_t RTCVideoEncoder::SetRates(uint32_t new_bit_rate, uint32_t frame_rate) {
- DVLOG(3) << __func__ << " new_bit_rate=" << new_bit_rate
+int32_t RTCVideoEncoder::SetRateAllocation(
+ const webrtc::VideoBitrateAllocation& allocation,
+ uint32_t frame_rate) {
+ DVLOG(3) << __func__ << " new_bit_rate=" << allocation.ToString()
<< ", frame_rate=" << frame_rate;
if (!impl_.get()) {
DVLOG(3) << "Encoder is not initialized";
@@ -939,7 +962,7 @@
gpu_task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&RTCVideoEncoder::Impl::RequestEncodingParametersChange,
- impl_, new_bit_rate, frame_rate));
+ impl_, allocation, frame_rate));
return WEBRTC_VIDEO_CODEC_OK;
}
diff --git a/content/renderer/media/webrtc/rtc_video_encoder.h b/content/renderer/media/webrtc/rtc_video_encoder.h
index 3116e16..57ffd31 100644
--- a/content/renderer/media/webrtc/rtc_video_encoder.h
+++ b/content/renderer/media/webrtc/rtc_video_encoder.h
@@ -18,6 +18,7 @@
#include "base/time/time.h"
#include "content/common/content_export.h"
#include "media/base/video_decoder_config.h"
+#include "third_party/webrtc/api/video/video_bitrate_allocation.h"
#include "third_party/webrtc/modules/video_coding/include/video_codec_interface.h"
#include "ui/gfx/geometry/size.h"
@@ -57,7 +58,8 @@
webrtc::EncodedImageCallback* callback) override;
int32_t Release() override;
int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override;
- int32_t SetRates(uint32_t new_bit_rate, uint32_t frame_rate) override;
+ int32_t SetRateAllocation(const webrtc::VideoBitrateAllocation& allocation,
+ uint32_t framerate) override;
bool SupportsNativeHandle() const override;
const char* ImplementationName() const override;
diff --git a/media/base/BUILD.gn b/media/base/BUILD.gn
index ad4a75d..f0fa3368 100644
--- a/media/base/BUILD.gn
+++ b/media/base/BUILD.gn
@@ -251,6 +251,8 @@
"unaligned_shared_memory.h",
"user_input_monitor.cc",
"user_input_monitor.h",
+ "video_bitrate_allocation.cc",
+ "video_bitrate_allocation.h",
"video_codecs.cc",
"video_codecs.h",
"video_color_space.cc",
@@ -496,6 +498,7 @@
"unaligned_shared_memory_unittest.cc",
"user_input_monitor_unittest.cc",
"vector_math_unittest.cc",
+ "video_bitrate_allocation_unittest.cc",
"video_codecs_unittest.cc",
"video_color_space_unittest.cc",
"video_decoder_config_unittest.cc",
diff --git a/media/base/video_bitrate_allocation.cc b/media/base/video_bitrate_allocation.cc
new file mode 100644
index 0000000..629092e
--- /dev/null
+++ b/media/base/video_bitrate_allocation.cc
@@ -0,0 +1,54 @@
+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "video_bitrate_allocation.h"
+
+#include <limits>
+#include <numeric>
+
+#include "base/logging.h"
+
+namespace media {
+
+constexpr size_t VideoBitrateAllocation::kMaxSpatialLayers;
+constexpr size_t VideoBitrateAllocation::kMaxTemporalLayers;
+
+VideoBitrateAllocation::VideoBitrateAllocation() : bitrates_{} {}
+
+bool VideoBitrateAllocation::SetBitrate(size_t spatial_index,
+ size_t temporal_index,
+ int bitrate_bps) {
+ CHECK_LT(spatial_index, kMaxSpatialLayers);
+ CHECK_LT(temporal_index, kMaxTemporalLayers);
+ CHECK_GE(bitrate_bps, 0);
+
+ if (GetSumBps() - bitrates_[spatial_index][temporal_index] >
+ std::numeric_limits<int>::max() - bitrate_bps) {
+ return false; // Would cause overflow of the sum.
+ }
+
+ bitrates_[spatial_index][temporal_index] = bitrate_bps;
+ return true;
+}
+
+int VideoBitrateAllocation::GetBitrateBps(size_t spatial_index,
+ size_t temporal_index) const {
+ CHECK_LT(spatial_index, kMaxSpatialLayers);
+ CHECK_LT(temporal_index, kMaxTemporalLayers);
+ return bitrates_[spatial_index][temporal_index];
+}
+
+int VideoBitrateAllocation::GetSumBps() const {
+ int sum = 0;
+ for (size_t spatial_index = 0; spatial_index < kMaxSpatialLayers;
+ ++spatial_index) {
+ for (size_t temporal_index = 0; temporal_index < kMaxTemporalLayers;
+ ++temporal_index) {
+ sum += bitrates_[spatial_index][temporal_index];
+ }
+ }
+ return sum;
+}
+
+} // namespace media
diff --git a/media/base/video_bitrate_allocation.h b/media/base/video_bitrate_allocation.h
new file mode 100644
index 0000000..6341ce3f
--- /dev/null
+++ b/media/base/video_bitrate_allocation.h
@@ -0,0 +1,43 @@
+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_BASE_VIDEO_BITRATE_ALLOCATION_H_
+#define MEDIA_BASE_VIDEO_BITRATE_ALLOCATION_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "base/macros.h"
+#include "media/base/media_export.h"
+
+namespace media {
+
+// Class that describes how video bitrate, in bps, is allocated across temporal
+// and spatial layers. Not that bitrates are NOT cumulative. Depending on if
+// layers are dependent or not, it is up to the user to aggregate.
+class MEDIA_EXPORT VideoBitrateAllocation {
+ public:
+ static constexpr size_t kMaxSpatialLayers = 5;
+ static constexpr size_t kMaxTemporalLayers = 4;
+
+ VideoBitrateAllocation();
+ ~VideoBitrateAllocation() = default;
+
+ // Returns if this bitrate can't be set (sum exceeds int max value).
+ bool SetBitrate(size_t spatial_index, size_t temporal_index, int bitrate_bps);
+
+ // Returns the bitrate for specified spatial/temporal index, or 0 if not set.
+ int GetBitrateBps(size_t spatial_index, size_t temporal_index) const;
+
+ // Sum of all bitrates.
+ int32_t GetSumBps() const;
+
+ private:
+ int bitrates_[kMaxSpatialLayers][kMaxTemporalLayers];
+ DISALLOW_COPY_AND_ASSIGN(VideoBitrateAllocation);
+};
+
+} // namespace media
+
+#endif // MEDIA_BASE_VIDEO_BITRATE_ALLOCATION_H_
diff --git a/media/base/video_bitrate_allocation_unittest.cc b/media/base/video_bitrate_allocation_unittest.cc
new file mode 100644
index 0000000..ce9e8ac
--- /dev/null
+++ b/media/base/video_bitrate_allocation_unittest.cc
@@ -0,0 +1,70 @@
+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <set>
+
+#include "base/logging.h"
+#include "media/base/video_bitrate_allocation.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace media {
+
+TEST(VideoBitrateAllocationTest, SetAndGet) {
+ int sum = 0;
+ int layer_rate = 0;
+ VideoBitrateAllocation allocation;
+ for (size_t spatial_index = 0;
+ spatial_index < VideoBitrateAllocation::kMaxSpatialLayers;
+ ++spatial_index) {
+ for (size_t temporal_index = 0;
+ temporal_index < VideoBitrateAllocation::kMaxTemporalLayers;
+ ++temporal_index) {
+ sum += layer_rate;
+ EXPECT_TRUE(
+ allocation.SetBitrate(spatial_index, temporal_index, layer_rate++));
+ }
+ }
+ EXPECT_EQ(sum, allocation.GetSumBps());
+
+ layer_rate = 0;
+ for (size_t spatial_index = 0;
+ spatial_index < VideoBitrateAllocation::kMaxSpatialLayers;
+ ++spatial_index) {
+ for (size_t temporal_index = 0;
+ temporal_index < VideoBitrateAllocation::kMaxTemporalLayers;
+ ++temporal_index) {
+ EXPECT_EQ(allocation.GetBitrateBps(spatial_index, temporal_index),
+ layer_rate++);
+ }
+ }
+}
+
+TEST(VideoBitrateAllocationTest, CanSetMaxValue) {
+ VideoBitrateAllocation allocation;
+ // Single cell containing max value.
+ EXPECT_TRUE(allocation.SetBitrate(0, 0, std::numeric_limits<int>::max()));
+ // Setting to 0 is OK. Doesn't increase sum.
+ EXPECT_TRUE(allocation.SetBitrate(0, 1, 0));
+ // Adding 1 will overflow.
+ EXPECT_FALSE(allocation.SetBitrate(0, 2, 1));
+
+ EXPECT_EQ(std::numeric_limits<int>::max(), allocation.GetSumBps());
+}
+
+TEST(VideoBitrateAllocationTest, ValidatesSumWhenOverwriting) {
+ VideoBitrateAllocation allocation;
+ // Fill up to max sum.
+ EXPECT_TRUE(allocation.SetBitrate(0, 0, std::numeric_limits<int>::max() - 2));
+ EXPECT_TRUE(allocation.SetBitrate(0, 1, 2));
+ // This will overflow, old value kept.
+ EXPECT_FALSE(allocation.SetBitrate(0, 1, 3));
+ EXPECT_EQ(allocation.GetBitrateBps(0, 1), 2);
+ // OK only since we subtract the previous 2.
+ EXPECT_TRUE(allocation.SetBitrate(0, 1, 1));
+ EXPECT_EQ(allocation.GetBitrateBps(0, 1), 1);
+
+ EXPECT_EQ(std::numeric_limits<int>::max() - 1, allocation.GetSumBps());
+}
+
+} // namespace media
diff --git a/media/video/video_encode_accelerator.cc b/media/video/video_encode_accelerator.cc
index 4b344b7..1f6c0dcb 100644
--- a/media/video/video_encode_accelerator.cc
+++ b/media/video/video_encode_accelerator.cc
@@ -24,6 +24,12 @@
std::move(flush_callback).Run(false);
}
+void VideoEncodeAccelerator::RequestEncodingParametersChange(
+ const VideoBitrateAllocation& bitrate,
+ uint32_t framerate) {
+ RequestEncodingParametersChange(bitrate.GetSumBps(), framerate);
+}
+
} // namespace media
namespace std {
diff --git a/media/video/video_encode_accelerator.h b/media/video/video_encode_accelerator.h
index 0162dde..ddc1d77 100644
--- a/media/video/video_encode_accelerator.h
+++ b/media/video/video_encode_accelerator.h
@@ -17,6 +17,7 @@
#include "base/single_thread_task_runner.h"
#include "media/base/bitstream_buffer.h"
#include "media/base/media_export.h"
+#include "media/base/video_bitrate_allocation.h"
#include "media/base/video_decoder_config.h"
#include "media/base/video_frame.h"
@@ -142,7 +143,7 @@
// |buffer| is the bitstream buffer to use for output.
virtual void UseOutputBitstreamBuffer(const BitstreamBuffer& buffer) = 0;
- // Request a change to the encoding parameters. This is only a request,
+ // Request a change to the encoding parameters. This is only a request,
// fulfilled on a best-effort basis.
// Parameters:
// |bitrate| is the requested new bitrate, in bits per second.
@@ -150,6 +151,16 @@
virtual void RequestEncodingParametersChange(uint32_t bitrate,
uint32_t framerate) = 0;
+ // Request a change to the encoding parameters. This is only a request,
+ // fulfilled on a best-effort basis. If not implemented, default behavior is
+ // to get the sum over layers and pass to version with bitrate as uint32_t.
+ // Parameters:
+ // |bitrate| is the requested new bitrate, per spatial and temporal layer.
+ // |framerate| is the requested new framerate, in frames per second.
+ virtual void RequestEncodingParametersChange(
+ const VideoBitrateAllocation& bitrate,
+ uint32_t framerate);
+
// Destroys the encoder: all pending inputs and outputs are dropped
// immediately and the component is freed. This call may asynchronously free
// system resources, but its client-visible effects are synchronous. After
