Speed up chromoting encoding and decoding path for VP8
Several enhancements in this patch:
1. Encoder reports updated rects
2. VP8 decoder only performs YUV conversion on updated rects
3. Painting only updated rects in pepper plugin
BUG=71253
TEST=Use chromoting to watch video
Review URL: http://codereview.chromium.org/6368070
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@73737 0039d316-1c4b-4281-b951-d872f2087c98
diff --git a/remoting/base/decoder_vp8.cc b/remoting/base/decoder_vp8.cc
index b150c4b2..d7744a8 100644
--- a/remoting/base/decoder_vp8.cc
+++ b/remoting/base/decoder_vp8.cc
@@ -1,4 +1,4 @@
-// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
@@ -89,14 +89,38 @@
stride = -stride;
}
- media::ConvertYUVToRGB32(image->planes[0], image->planes[1], image->planes[2],
- data_start, frame_->width(), frame_->height(),
- image->stride[0], image->stride[1], stride,
- media::YV12);
+ // Propogate updated rects.
+ updated_rects_.clear();
+ for (int i = 0; i < packet->dirty_rects_size(); ++i) {
+ gfx::Rect r = gfx::Rect(packet->dirty_rects(i).x(),
+ packet->dirty_rects(i).y(),
+ packet->dirty_rects(i).width(),
+ packet->dirty_rects(i).height());
+
+ // Perform color space conversion only on the updated rectangle.
+ ConvertYUVToRGB32WithRect(image->planes[0],
+ image->planes[1],
+ image->planes[2],
+ data_start,
+ r.x(),
+ r.y(),
+ r.width(),
+ r.height(),
+ image->stride[0],
+ image->stride[1],
+ stride);
+
+ // Since the image generated by client is upside-down we need to report
+ // updated rects as upside-down.
+ if (reverse_rows_)
+ r.set_y(image->d_h - r.bottom());
+ updated_rects_.push_back(r);
+ }
return DECODE_DONE;
}
void DecoderVp8::GetUpdatedRects(UpdatedRects* rects) {
+ rects->swap(updated_rects_);
}
void DecoderVp8::Reset() {
diff --git a/remoting/base/decoder_vp8.h b/remoting/base/decoder_vp8.h
index a59a601..6e15a0b6 100644
--- a/remoting/base/decoder_vp8.h
+++ b/remoting/base/decoder_vp8.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
@@ -42,6 +42,9 @@
vpx_codec_ctx_t* codec_;
+ // Record the updated rects in the last decode.
+ UpdatedRects updated_rects_;
+
DISALLOW_COPY_AND_ASSIGN(DecoderVp8);
};
diff --git a/remoting/base/encoder_vp8.cc b/remoting/base/encoder_vp8.cc
index 7ac75d3..fe12110 100644
--- a/remoting/base/encoder_vp8.cc
+++ b/remoting/base/encoder_vp8.cc
@@ -1,4 +1,4 @@
-// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
@@ -8,6 +8,7 @@
#include "media/base/callback.h"
#include "media/base/yuv_convert.h"
#include "remoting/base/capture_data.h"
+#include "remoting/base/util.h"
#include "remoting/proto/video.pb.h"
extern "C" {
@@ -94,7 +95,8 @@
return r;
}
-bool EncoderVp8::PrepareImage(scoped_refptr<CaptureData> capture_data) {
+bool EncoderVp8::PrepareImage(scoped_refptr<CaptureData> capture_data,
+ std::vector<gfx::Rect>* updated_rects) {
const int plane_size = capture_data->width() * capture_data->height();
if (!yuv_image_.get()) {
@@ -138,21 +140,23 @@
const int y_stride = image_->stride[0];
const int uv_stride = image_->stride[1];
+ DCHECK(updated_rects->empty());
for (InvalidRects::const_iterator r = rects.begin(); r != rects.end(); ++r) {
+ // Align the rectangle report it as updated.
gfx::Rect rect = AlignRect(*r, image_->w, image_->h);
- int in_offset = in_stride * rect.y() + 4 * rect.x();
- int y_offset = y_stride * rect.y() + rect.x();
- int uv_offset = (uv_stride * rect.y() + rect.x()) / 2;
+ updated_rects->push_back(rect);
- media::ConvertRGB32ToYUV(in + in_offset,
- y_out + y_offset,
- u_out + uv_offset,
- v_out + uv_offset,
- rect.width(),
- rect.height(),
- in_stride,
- y_stride,
- uv_stride);
+ ConvertRGB32ToYUVWithRect(in,
+ y_out,
+ u_out,
+ v_out,
+ rect.x(),
+ rect.y(),
+ rect.width(),
+ rect.height(),
+ in_stride,
+ y_stride,
+ uv_stride);
}
return true;
}
@@ -167,7 +171,8 @@
initialized_ = ret;
}
- if (!PrepareImage(capture_data)) {
+ std::vector<gfx::Rect> updated_rects;
+ if (!PrepareImage(capture_data, &updated_rects)) {
NOTREACHED() << "Can't image data for encoding";
}
@@ -211,6 +216,13 @@
message->mutable_format()->set_encoding(VideoPacketFormat::ENCODING_VP8);
message->set_flags(VideoPacket::FIRST_PACKET | VideoPacket::LAST_PACKET |
VideoPacket::LAST_PARTITION);
+ for (size_t i = 0; i < updated_rects.size(); ++i) {
+ Rect* rect = message->add_dirty_rects();
+ rect->set_x(updated_rects[i].x());
+ rect->set_y(updated_rects[i].y());
+ rect->set_width(updated_rects[i].width());
+ rect->set_height(updated_rects[i].height());
+ }
data_available_callback->Run(message);
delete data_available_callback;
diff --git a/remoting/base/encoder_vp8.h b/remoting/base/encoder_vp8.h
index 6c3357c9..8a11cda 100644
--- a/remoting/base/encoder_vp8.h
+++ b/remoting/base/encoder_vp8.h
@@ -1,10 +1,13 @@
-// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef REMOTING_BASE_ENCODER_VP8_H_
#define REMOTING_BASE_ENCODER_VP8_H_
+#include <vector>
+
+#include "gfx/rect.h"
#include "remoting/base/encoder.h"
typedef struct vpx_codec_ctx vpx_codec_ctx_t;
@@ -26,8 +29,10 @@
// Initialize the encoder. Returns true if successful.
bool Init(int width, int height);
- // Prepare |image_| for encoding. Returns true if successful.
- bool PrepareImage(scoped_refptr<CaptureData> capture_data);
+ // Prepare |image_| for encoding. Write updated rectangles into
+ // |updated_rects|. Returns true if successful.
+ bool PrepareImage(scoped_refptr<CaptureData> capture_data,
+ std::vector<gfx::Rect>* updated_rects);
// True if the encoder is initialized.
bool initialized_;
diff --git a/remoting/base/util.cc b/remoting/base/util.cc
index 5cf70fe..9341050 100644
--- a/remoting/base/util.cc
+++ b/remoting/base/util.cc
@@ -1,7 +1,9 @@
-// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
+#include "media/base/video_frame.h"
+#include "media/base/yuv_convert.h"
#include "remoting/base/util.h"
#include "base/logging.h"
@@ -25,4 +27,70 @@
}
}
+// Helper methods to calculate plane offset given the coordinates.
+static int CalculateRGBOffset(int x, int y, int stride) {
+ return stride * y + GetBytesPerPixel(media::VideoFrame::RGB32) * x;
+}
+
+static int CalculateYOffset(int x, int y, int stride) {
+ return stride * y + x;
+}
+
+static int CalculateUVOffset(int x, int y, int stride) {
+ return stride * y / 2 + x / 2;
+}
+
+void ConvertYUVToRGB32WithRect(const uint8* y_plane,
+ const uint8* u_plane,
+ const uint8* v_plane,
+ uint8* rgb_plane,
+ int x,
+ int y,
+ int width,
+ int height,
+ int y_stride,
+ int uv_stride,
+ int rgb_stride) {
+ int rgb_offset = CalculateRGBOffset(x, y, rgb_stride);
+ int y_offset = CalculateYOffset(x, y, y_stride);
+ int uv_offset = CalculateUVOffset(x, y, uv_stride);;
+
+ media::ConvertYUVToRGB32(y_plane + y_offset,
+ u_plane + uv_offset,
+ v_plane + uv_offset,
+ rgb_plane + rgb_offset,
+ width,
+ height,
+ y_stride,
+ uv_stride,
+ rgb_stride,
+ media::YV12);
+}
+
+void ConvertRGB32ToYUVWithRect(const uint8* rgb_plane,
+ uint8* y_plane,
+ uint8* u_plane,
+ uint8* v_plane,
+ int x,
+ int y,
+ int width,
+ int height,
+ int rgb_stride,
+ int y_stride,
+ int uv_stride) {
+ int rgb_offset = CalculateRGBOffset(x, y, rgb_stride);
+ int y_offset = CalculateYOffset(x, y, y_stride);
+ int uv_offset = CalculateUVOffset(x, y, uv_stride);;
+
+ media::ConvertRGB32ToYUV(rgb_plane + rgb_offset,
+ y_plane + y_offset,
+ u_plane + uv_offset,
+ v_plane + uv_offset,
+ width,
+ height,
+ rgb_stride,
+ y_stride,
+ uv_stride);
+}
+
} // namespace remoting
diff --git a/remoting/base/util.h b/remoting/base/util.h
index bf5cff5..51a8d10 100644
--- a/remoting/base/util.h
+++ b/remoting/base/util.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Copyright (c) 2011 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
@@ -9,9 +9,34 @@
namespace remoting {
-// TODO(sergeyu): Move this to media::VideoFrame.
+// TODO(sergeyu): Move these methods to media.
int GetBytesPerPixel(media::VideoFrame::Format format);
+// Convert YUV to RGB32 on a specific rectangle.
+void ConvertYUVToRGB32WithRect(const uint8* y_plane,
+ const uint8* u_plane,
+ const uint8* v_plane,
+ uint8* rgb_plane,
+ int x,
+ int y,
+ int width,
+ int height,
+ int y_stride,
+ int uv_stride,
+ int rgb_stride);
+
+void ConvertRGB32ToYUVWithRect(const uint8* rgb_plane,
+ uint8* y_plane,
+ uint8* u_plane,
+ uint8* v_plane,
+ int x,
+ int y,
+ int width,
+ int height,
+ int rgb_stride,
+ int y_stride,
+ int uv_stride);
+
} // namespace remoting
#endif // REMOTING_BASE_UTIL_H_
diff --git a/remoting/client/plugin/pepper_view.cc b/remoting/client/plugin/pepper_view.cc
index 4c58ed6..007de6d 100644
--- a/remoting/client/plugin/pepper_view.cc
+++ b/remoting/client/plugin/pepper_view.cc
@@ -10,6 +10,7 @@
#include "ppapi/cpp/point.h"
#include "ppapi/cpp/size.h"
#include "remoting/base/tracer.h"
+#include "remoting/base/util.h"
#include "remoting/client/client_context.h"
#include "remoting/client/plugin/chromoting_instance.h"
#include "remoting/client/plugin/pepper_util.h"
@@ -81,34 +82,41 @@
DCHECK(instance_->CurrentlyOnPluginThread());
TraceContext::tracer()->PrintString("Start Paint Frame.");
- // TODO(ajwong): We're assuming the native format is BGRA_PREMUL below. This
- // is wrong.
- pp::ImageData image(instance_, pp::ImageData::GetNativeImageDataFormat(),
- pp::Size(viewport_width_, viewport_height_),
- false);
- if (image.is_null()) {
- LOG(ERROR) << "Unable to allocate image of size: "
- << frame->width() << "x" << frame->height();
- return;
- }
- uint32_t* frame_data =
- reinterpret_cast<uint32_t*>(frame->data(media::VideoFrame::kRGBPlane));
- int frame_width = static_cast<int>(frame->width());
- int frame_height = static_cast<int>(frame->height());
- int max_height = std::min(frame_height, image.size().height());
- int max_width = std::min(frame_width, image.size().width());
- for (int y = 0; y < max_height; y++) {
- for (int x = 0; x < max_width; x++) {
- // Force alpha to be set to 255.
- *image.GetAddr32(pp::Point(x, y)) =
- frame_data[y*frame_width + x] | 0xFF000000;
+ uint8* frame_data = frame->data(media::VideoFrame::kRGBPlane);
+ const int kFrameStride = frame->stride(media::VideoFrame::kRGBPlane);
+ const int kBytesPerPixel = GetBytesPerPixel(media::VideoFrame::RGB32);
+
+ for (size_t i = 0; i < rects->size(); ++i) {
+ // TODO(ajwong): We're assuming the native format is BGRA_PREMUL below. This
+ // is wrong.
+ const gfx::Rect& r = (*rects)[i];
+
+ // TODO(hclam): Make sure rectangles are valid.
+ if (r.width() <= 0 || r.height() <= 0)
+ continue;
+
+ pp::ImageData image(instance_, pp::ImageData::GetNativeImageDataFormat(),
+ pp::Size(r.width(), r.height()),
+ false);
+ if (image.is_null()) {
+ LOG(ERROR) << "Unable to allocate image of size: "
+ << r.width() << "x" << r.height();
+ return;
}
+
+ // Copy pixel data into |image|.
+ uint8* in = frame_data + kFrameStride * r.y() + kBytesPerPixel * r.x();
+ uint8* out = reinterpret_cast<uint8*>(image.data());
+ for (int j = 0; j < r.height(); ++j) {
+ memcpy(out, in, r.width() * kBytesPerPixel);
+ in += kFrameStride;
+ out += image.stride();
+ }
+
+ graphics2d_.PaintImageData(image, pp::Point(r.x(), r.y()));
}
- // For ReplaceContents, make sure the image size matches the device context
- // size! Otherwise, this will just silently do nothing.
- graphics2d_.ReplaceContents(&image);
graphics2d_.Flush(TaskToCompletionCallback(
task_factory_.NewRunnableMethod(&PepperView::OnPaintDone)));
diff --git a/remoting/proto/video.proto b/remoting/proto/video.proto
index 394f891..2dadcb8 100644
--- a/remoting/proto/video.proto
+++ b/remoting/proto/video.proto
@@ -32,6 +32,14 @@
optional Encoding encoding = 5 [default = ENCODING_INVALID];
}
+// TODO(hclam): Remove this message once we can obtain dirty rects from libvpx.
+message Rect {
+ optional int32 x = 1;
+ optional int32 y = 2;
+ optional int32 width = 3;
+ optional int32 height = 4;
+}
+
message VideoPacket {
// Bitmasks for use in the flags field below.
//
@@ -67,4 +75,10 @@
optional VideoPacketFormat format = 4;
optional bytes data = 5;
+
+ // This field is only for VP8 to provide out-of-band information of dirty
+ // rects.
+ // TODO(hclam): Remove this field when we can obtain this information from
+ // libvpx.
+ repeated Rect dirty_rects = 6;
}
