2
// Copyright 2010 Google Inc.
4
// Redistribution and use in source and binary forms, with or without
5
// modification, are permitted provided that the following conditions are met:
7
// 1. Redistributions of source code must retain the above copyright notice,
8
// this list of conditions and the following disclaimer.
9
// 2. Redistributions in binary form must reproduce the above copyright notice,
10
// this list of conditions and the following disclaimer in the documentation
11
// and/or other materials provided with the distribution.
12
// 3. The name of the author may not be used to endorse or promote products
13
// derived from this software without specific prior written permission.
15
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
16
// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
17
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
18
// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
19
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21
// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22
// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23
// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
24
// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
#include "talk/session/phone/videoadapter.h"
30
#include "talk/base/logging.h"
31
#include "talk/base/timeutils.h"
32
#include "talk/session/phone/videoframe.h"
36
// TODO: Make downgrades settable
37
static const int kMaxCpuDowngrades = 2; // Downgrade at most 2 times for CPU.
38
static const int kDefaultDowngradeWaitTimeMs = 2000;
40
// Cpu system load thresholds relative to max cpus.
41
static const float kHighSystemThreshold = 0.95f;
42
static const float kLowSystemThreshold = 0.75f;
44
// Cpu process load thresholds relative to current cpus.
45
static const float kMediumProcessThreshold = 0.50f;
47
// TODO: Consider making scale factor table settable, to allow
48
// application to select quality vs performance tradeoff.
49
// List of scale factors that adapter will scale by.
50
#if defined(IOS) || defined(ANDROID)
51
// Mobile needs 1/4 scale for VGA (640x360) to QQVGA (160x90)
52
// or 1/4 scale for HVGA (480x270) to QQHVGA (120x67)
53
static const int kMinNumPixels = 120 * 67;
54
static float kScaleFactors[] = {
62
// PC needs 1/8 scale for HD (1280x720) to QQVGA (160x90)
63
static const int kMinNumPixels = 160 * 100;
64
static float kScaleFactors[] = {
70
3.f/16.f, // 3/16 scale
75
// Find scale factor that applied to width and height, is best match
77
float VideoAdapter::FindClosestScale(int width, int height,
78
int target_num_pixels) {
79
if (!target_num_pixels) {
82
int best_distance = INT_MAX;
83
int best_index = 0; // default to unscaled
84
for (size_t i = 0u; i < ARRAY_SIZE(kScaleFactors); ++i) {
85
int test_num_pixels = static_cast<int>(width * kScaleFactors[i] *
86
height * kScaleFactors[i]);
87
int diff = test_num_pixels - target_num_pixels;
91
if (diff < best_distance) {
94
if (!best_distance) { // Found exact match
99
return kScaleFactors[best_index];
102
// There are several frame sizes used by Adapter. This explains them
103
// input_format - set once by server to frame size expected from the camera.
104
// output_format - size that output would like to be. Includes framerate.
105
// output_num_pixels - size that output should be constrained to. Used to
106
// compute output_format from in_frame.
107
// in_frame - actual camera captured frame size, which is typically the same
108
// as input_format. This can also be rotated or cropped for aspect ratio.
109
// out_frame - actual frame output by adapter. Should be a direct scale of
110
// in_frame maintaining rotation and aspect ratio.
111
// OnOutputFormatRequest - server requests you send this resolution based on
113
// OnEncoderResolutionRequest - encoder requests you send this resolution based
115
// OnCpuLoadUpdated - cpu monitor requests you send this resolution based on
118
///////////////////////////////////////////////////////////////////////
119
// Implementation of VideoAdapter
120
VideoAdapter::VideoAdapter()
121
: output_num_pixels_(0),
122
black_output_(false),
124
drop_frame_count_(0) {
127
VideoAdapter::~VideoAdapter() {
130
// TODO: Consider SetInputFormat and SetOutputFormat without
132
void VideoAdapter::SetInputFormat(const VideoFormat& format) {
133
talk_base::CritScope cs(&critical_section_);
134
input_format_ = format;
135
output_format_.interval = talk_base::_max(
136
output_format_.interval, input_format_.interval);
139
void VideoAdapter::SetOutputFormat(const VideoFormat& format) {
140
talk_base::CritScope cs(&critical_section_);
141
output_format_ = format;
142
output_num_pixels_ = output_format_.width * output_format_.height;
143
output_format_.interval = talk_base::_max(
144
output_format_.interval, input_format_.interval);
145
drop_frame_count_ = 0;
148
const VideoFormat& VideoAdapter::input_format() {
149
talk_base::CritScope cs(&critical_section_);
150
return input_format_;
153
const VideoFormat& VideoAdapter::output_format() {
154
talk_base::CritScope cs(&critical_section_);
155
return output_format_;
158
void VideoAdapter::SetBlackOutput(bool black) {
159
talk_base::CritScope cs(&critical_section_);
160
black_output_ = black;
163
// Constrain output resolution to this many pixels overall
164
void VideoAdapter::SetOutputNumPixels(int num_pixels) {
165
output_num_pixels_ = num_pixels;
168
int VideoAdapter::GetOutputNumPixels() const {
169
return output_num_pixels_;
172
bool VideoAdapter::AdaptFrame(const VideoFrame* in_frame,
173
const VideoFrame** out_frame) {
174
talk_base::CritScope cs(&critical_section_);
176
if (!in_frame || !out_frame || input_format_.IsSize0x0()) {
180
// Drop the input frame if necessary.
181
bool should_drop = false;
182
if (!output_num_pixels_) {
183
// Drop all frames as the output format is 0x0.
186
// Drop some frames based on the ratio of the input fps and the output fps.
187
// We assume that the output fps is a factor of the input fps. In other
188
// words, the output interval is divided by the input interval evenly.
189
should_drop = (drop_frame_count_ > 0);
190
if (input_format_.interval > 0 &&
191
output_format_.interval > input_format_.interval) {
193
drop_frame_count_ %= output_format_.interval / input_format_.interval;
197
if (output_num_pixels_) {
198
float scale = VideoAdapter::FindClosestScale(in_frame->GetWidth(),
199
in_frame->GetHeight(),
201
output_format_.width = static_cast<int>(in_frame->GetWidth() * scale);
202
output_format_.height = static_cast<int>(in_frame->GetHeight() * scale);
210
if (!StretchToOutputFrame(in_frame)) {
214
*out_frame = output_frame_.get();
218
bool VideoAdapter::StretchToOutputFrame(const VideoFrame* in_frame) {
219
int output_width = output_format_.width;
220
int output_height = output_format_.height;
222
// Create and stretch the output frame if it has not been created yet or its
223
// size is not same as the expected.
224
bool stretched = false;
225
if (!output_frame_.get() ||
226
output_frame_->GetWidth() != static_cast<size_t>(output_width) ||
227
output_frame_->GetHeight() != static_cast<size_t>(output_height)) {
229
in_frame->Stretch(output_width, output_height, true, true));
230
if (!output_frame_.get()) {
231
LOG(LS_WARNING) << "Adapter failed to stretch frame to "
232
<< output_width << "x" << output_height;
239
if (!black_output_) {
241
// The output frame does not need to be blacken and has not been stretched
242
// from the input frame yet, stretch the input frame. This is the most
244
in_frame->StretchToFrame(output_frame_.get(), true, true);
249
output_frame_->SetToBlack();
252
output_frame_->SetElapsedTime(in_frame->GetElapsedTime());
253
output_frame_->SetTimeStamp(in_frame->GetTimeStamp());
259
///////////////////////////////////////////////////////////////////////
260
// Implementation of CoordinatedVideoAdapter
261
CoordinatedVideoAdapter::CoordinatedVideoAdapter()
262
: cpu_adaptation_(false),
263
gd_adaptation_(true),
264
view_adaptation_(true),
265
cpu_downgrade_count_(0),
266
cpu_downgrade_wait_time_(0),
267
view_desired_num_pixels_(INT_MAX),
268
view_desired_interval_(0),
269
encoder_desired_num_pixels_(INT_MAX),
270
cpu_desired_num_pixels_(INT_MAX) {
273
// Helper function to UPGRADE or DOWNGRADE a number of pixels
274
void CoordinatedVideoAdapter::StepPixelCount(
275
CoordinatedVideoAdapter::AdaptRequest request,
278
case CoordinatedVideoAdapter::DOWNGRADE:
282
case CoordinatedVideoAdapter::UPGRADE:
286
default: // No change in pixel count
292
// Find the adaptation request of the cpu based on the load. Return UPGRADE if
293
// the load is low, DOWNGRADE if the load is high, and KEEP otherwise.
294
CoordinatedVideoAdapter::AdaptRequest CoordinatedVideoAdapter::FindCpuRequest(
295
int current_cpus, int max_cpus,
296
float process_load, float system_load) {
297
// Downgrade if system is high and plugin is at least more than midrange.
298
if (system_load >= kHighSystemThreshold * max_cpus &&
299
process_load >= kMediumProcessThreshold * current_cpus) {
300
return CoordinatedVideoAdapter::DOWNGRADE;
301
// Upgrade if system is low.
302
} else if (system_load < kLowSystemThreshold * max_cpus) {
303
return CoordinatedVideoAdapter::UPGRADE;
305
return CoordinatedVideoAdapter::KEEP;
308
// A remote view request for a new resolution.
309
void CoordinatedVideoAdapter::OnOutputFormatRequest(const VideoFormat& format) {
310
talk_base::CritScope cs(&request_critical_section_);
311
if (!view_adaptation_) {
314
// Set output for initial aspect ratio in mediachannel unittests.
315
int old_num_pixels = GetOutputNumPixels();
316
SetOutputFormat(format);
317
SetOutputNumPixels(old_num_pixels);
318
view_desired_num_pixels_ = format.width * format.height;
319
view_desired_interval_ = format.interval;
320
bool changed = AdaptToMinimumFormat();
321
LOG(LS_INFO) << "VAdapt View Request: "
322
<< format.width << "x" << format.height
323
<< " Pixels: " << view_desired_num_pixels_
324
<< " Changed: " << (changed ? "true" : "false");
327
// A Bandwidth GD request for new resolution
328
void CoordinatedVideoAdapter::OnEncoderResolutionRequest(
329
int width, int height, AdaptRequest request) {
330
talk_base::CritScope cs(&request_critical_section_);
331
if (!gd_adaptation_) {
334
if (KEEP != request) {
335
int new_encoder_desired_num_pixels = width * height;
336
int old_num_pixels = GetOutputNumPixels();
337
if (new_encoder_desired_num_pixels != old_num_pixels) {
338
LOG(LS_VERBOSE) << "VAdapt GD resolution stale. Ignored";
340
// Update the encoder desired format based on the request.
341
encoder_desired_num_pixels_ = new_encoder_desired_num_pixels;
342
StepPixelCount(request, &encoder_desired_num_pixels_);
345
bool changed = AdaptToMinimumFormat();
346
LOG(LS_INFO) << "VAdapt GD Request: "
347
<< (DOWNGRADE == request ? "down" :
348
(UPGRADE == request ? "up" : "keep"))
349
<< " From: " << width << "x" << height
350
<< " Pixels: " << encoder_desired_num_pixels_
351
<< " Changed: " << (changed ? "true" : "false");
354
// A CPU request for new resolution
355
void CoordinatedVideoAdapter::OnCpuLoadUpdated(
356
int current_cpus, int max_cpus, float process_load, float system_load) {
357
talk_base::CritScope cs(&request_critical_section_);
358
if (!cpu_adaptation_) {
361
AdaptRequest request = FindCpuRequest(current_cpus, max_cpus,
362
process_load, system_load);
363
// Update how many times we have downgraded due to the cpu load.
366
if (cpu_downgrade_count_ < kMaxCpuDowngrades) {
367
// Ignore downgrades if we have downgraded the maximum times or we just
368
// downgraded in a short time.
369
if (cpu_downgrade_wait_time_ != 0 &&
370
talk_base::TimeIsLater(talk_base::Time(),
371
cpu_downgrade_wait_time_)) {
372
LOG(LS_VERBOSE) << "VAdapt CPU load high but do not downgrade until "
373
<< talk_base::TimeUntil(cpu_downgrade_wait_time_)
377
++cpu_downgrade_count_;
380
LOG(LS_VERBOSE) << "VAdapt CPU load high but do not downgrade "
381
"because maximum downgrades reached";
385
if (cpu_downgrade_count_ > 0) {
386
bool is_min = IsMinimumFormat(cpu_desired_num_pixels_);
388
--cpu_downgrade_count_;
390
LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade "
391
"because cpu is not limiting resolution";
394
LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade "
395
"because minimum downgrades reached";
402
if (KEEP != request) {
403
// TODO: compute stepping up/down from OutputNumPixels but
404
// clamp to inputpixels / 4 (2 steps)
405
cpu_desired_num_pixels_ = static_cast<int>(
406
input_format().width * input_format().height >> cpu_downgrade_count_);
408
bool changed = AdaptToMinimumFormat();
409
LOG(LS_INFO) << "VAdapt CPU Request: "
410
<< (DOWNGRADE == request ? "down" :
411
(UPGRADE == request ? "up" : "keep"))
412
<< " Process: " << process_load
413
<< " System: " << system_load
414
<< " Steps: " << cpu_downgrade_count_
415
<< " Changed: " << (changed ? "true" : "false");
418
// Called by cpu adapter on up requests.
419
bool CoordinatedVideoAdapter::IsMinimumFormat(int pixels) {
420
// Find closest scale factor that matches input resolution to min_num_pixels
421
// and set that for output resolution. This is not needed for VideoAdapter,
422
// but provides feedback to unittests and users on expected resolution.
423
// Actual resolution is based on input frame.
424
VideoFormat new_output = output_format();
425
VideoFormat input = input_format();
426
if (input_format().IsSize0x0()) {
430
if (!input.IsSize0x0()) {
431
scale = FindClosestScale(input.width,
435
new_output.width = static_cast<int>(input.width * scale);
436
new_output.height = static_cast<int>(input.height * scale);
437
int new_pixels = new_output.width * new_output.height;
438
int num_pixels = GetOutputNumPixels();
439
return new_pixels <= num_pixels;
442
// Called by all coordinators when there is a change.
443
bool CoordinatedVideoAdapter::AdaptToMinimumFormat() {
444
int old_num_pixels = GetOutputNumPixels();
445
// Get the min of the formats that the server, encoder, and cpu wants.
446
int min_num_pixels = view_desired_num_pixels_;
447
if (encoder_desired_num_pixels_ &&
448
(encoder_desired_num_pixels_ < min_num_pixels)) {
449
min_num_pixels = encoder_desired_num_pixels_;
451
if (cpu_adaptation_ && cpu_desired_num_pixels_ &&
452
(cpu_desired_num_pixels_ < min_num_pixels)) {
453
min_num_pixels = cpu_desired_num_pixels_;
454
// Update the cpu_downgrade_wait_time_ if we are going to downgrade video.
455
cpu_downgrade_wait_time_ =
456
talk_base::TimeAfter(kDefaultDowngradeWaitTimeMs);
458
// prevent going below QQVGA
459
if (min_num_pixels > 0 && min_num_pixels < kMinNumPixels) {
460
min_num_pixels = kMinNumPixels;
462
SetOutputNumPixels(min_num_pixels);
464
// Find closest scale factor that matches input resolution to min_num_pixels
465
// and set that for output resolution. This is not needed for VideoAdapter,
466
// but provides feedback to unittests and users on expected resolution.
467
// Actual resolution is based on input frame.
468
VideoFormat new_output = output_format();
469
VideoFormat input = input_format();
470
if (input_format().IsSize0x0()) {
474
if (!input.IsSize0x0()) {
475
scale = FindClosestScale(input.width,
479
new_output.width = static_cast<int>(input.width * scale);
480
new_output.height = static_cast<int>(input.height * scale);
481
new_output.interval = view_desired_interval_;
482
SetOutputFormat(new_output);
483
int new_num_pixels = GetOutputNumPixels();
484
bool changed = new_num_pixels != old_num_pixels;
486
LOG(LS_VERBOSE) << "VAdapt Status View: " << view_desired_num_pixels_
487
<< " GD: " << encoder_desired_num_pixels_
488
<< " CPU: " << cpu_desired_num_pixels_
489
<< " Pixels: " << min_num_pixels
490
<< " Scale: " << scale
491
<< " Resolution: " << new_output.width
492
<< "x" << new_output.height
493
<< " Changed: " << (changed ? "true" : "false");
497
} // namespace cricket
added
removed
protocols/jabber/googletalk/libjingle/talk/session/phone/videoadapter.cc
