session/phone/videoadapter.cc

// libjingle
// Copyright 2010 Google Inc.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//  1. Redistributions of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//  2. Redistributions in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//  3. The name of the author may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "talk/session/phone/videoadapter.h"

#include <limits.h>

#include "talk/base/logging.h"
#include "talk/base/timeutils.h"
#include "talk/session/phone/videoframe.h"

namespace cricket {

// TODO: Make downgrades settable
static const int kMaxCpuDowngrades = 2;  // Downgrade at most 2 times for CPU.
static const int kDefaultDowngradeWaitTimeMs = 2000;

// Cpu system load thresholds relative to max cpus.
static const float kHighSystemThreshold = 0.95f;
static const float kLowSystemThreshold = 0.75f;

// Cpu process load thresholds relative to current cpus.
static const float kMediumProcessThreshold = 0.50f;

// TODO: Consider making scale factor table settable, to allow
// application to select quality vs performance tradeoff.
// List of scale factors that adapter will scale by.
#if defined(IOS) || defined(ANDROID)
// Mobile needs 1/4 scale for VGA (640x360) to QQVGA (160x90)
// or 1/4 scale for HVGA (480x270) to QQHVGA (120x67)
static const int kMinNumPixels = 120 * 67;
static float kScaleFactors[] = {
  1.f, // full size
  3.f/4.f, // 3/4 scale
  1.f/2.f, // 1/2 scale
  3.f/8.f, // 3/8 scale
  1.f/4.f, // 1/4 scale
};
#else
// PC needs 1/8 scale for HD (1280x720) to QQVGA (160x90)
static const int kMinNumPixels = 160 * 100;
static float kScaleFactors[] = {
  1.f, // full size
  3.f/4.f, // 3/4 scale
  1.f/2.f, // 1/2 scale
  3.f/8.f, // 3/8 scale
  1.f/4.f, // 1/4 scale
  3.f/16.f, // 3/16 scale
  1.f/8.f // 1/8 scale
};
#endif

// Find scale factor that applied to width and height, is best match
// to num_pixels.
float VideoAdapter::FindClosestScale(int width, int height,
                                     int target_num_pixels) {
  if (!target_num_pixels) {
    return 0.f;
  }
  int best_distance = INT_MAX;
  int best_index = 0;  // default to unscaled
  for (size_t i = 0u; i < ARRAY_SIZE(kScaleFactors); ++i) {
    int test_num_pixels = static_cast<int>(width * kScaleFactors[i] *
                                           height * kScaleFactors[i]);
    int diff = test_num_pixels - target_num_pixels;
    if (diff < 0) {
      diff = -diff;
    }
    if (diff < best_distance) {
      best_distance = diff;
      best_index = i;
      if (!best_distance) { // Found exact match
        break;
      }
    }
  }
  return kScaleFactors[best_index];
}

// There are several frame sizes used by Adapter.  This explains them
// input_format - set once by server to frame size expected from the camera.
// output_format - size that output would like to be.  Includes framerate.
// output_num_pixels - size that output should be constrained to.  Used to
//   compute output_format from in_frame.
// in_frame - actual camera captured frame size, which is typically the same
//   as input_format.  This can also be rotated or cropped for aspect ratio.
// out_frame - actual frame output by adapter.  Should be a direct scale of
//   in_frame maintaining rotation and aspect ratio.
// OnOutputFormatRequest - server requests you send this resolution based on
//   view requests.
// OnEncoderResolutionRequest - encoder requests you send this resolution based
//   on bandwidth
// OnCpuLoadUpdated - cpu monitor requests you send this resolution based on
//   cpu load.

///////////////////////////////////////////////////////////////////////
// Implementation of VideoAdapter
VideoAdapter::VideoAdapter()
    : output_num_pixels_(0),
      black_output_(false),
      is_black_(false),
      drop_frame_count_(0) {
}

VideoAdapter::~VideoAdapter() {
}

// TODO: Consider SetInputFormat and SetOutputFormat without
// VideoFormat.
void VideoAdapter::SetInputFormat(const VideoFormat& format) {
  talk_base::CritScope cs(&critical_section_);
  input_format_ = format;
  output_format_.interval = talk_base::_max(
      output_format_.interval, input_format_.interval);
}

void VideoAdapter::SetOutputFormat(const VideoFormat& format) {
  talk_base::CritScope cs(&critical_section_);
  output_format_ = format;
  output_num_pixels_ = output_format_.width * output_format_.height;
  output_format_.interval = talk_base::_max(
      output_format_.interval, input_format_.interval);
  drop_frame_count_ = 0;
}

const VideoFormat& VideoAdapter::input_format() {
  talk_base::CritScope cs(&critical_section_);
  return input_format_;
}

const VideoFormat& VideoAdapter::output_format() {
  talk_base::CritScope cs(&critical_section_);
  return output_format_;
}

void VideoAdapter::SetBlackOutput(bool black) {
  talk_base::CritScope cs(&critical_section_);
  black_output_ = black;
}

// Constrain output resolution to this many pixels overall
void VideoAdapter::SetOutputNumPixels(int num_pixels) {
  output_num_pixels_ = num_pixels;
}

int VideoAdapter::GetOutputNumPixels() const {
  return output_num_pixels_;
}

bool VideoAdapter::AdaptFrame(const VideoFrame* in_frame,
                              const VideoFrame** out_frame) {
  talk_base::CritScope cs(&critical_section_);

  if (!in_frame || !out_frame || input_format_.IsSize0x0()) {
    return false;
  }

  // Drop the input frame if necessary.
  bool should_drop = false;
  if (!output_num_pixels_) {
    // Drop all frames as the output format is 0x0.
    should_drop = true;
  } else {
    // Drop some frames based on the ratio of the input fps and the output fps.
    // We assume that the output fps is a factor of the input fps. In other
    // words, the output interval is divided by the input interval evenly.
    should_drop = (drop_frame_count_ > 0);
    if (input_format_.interval > 0 &&
        output_format_.interval > input_format_.interval) {
      ++drop_frame_count_;
      drop_frame_count_ %= output_format_.interval / input_format_.interval;
    }
  }

  if (output_num_pixels_) {
    float scale = VideoAdapter::FindClosestScale(in_frame->GetWidth(),
                                                 in_frame->GetHeight(),
                                                 output_num_pixels_);
    output_format_.width = static_cast<int>(in_frame->GetWidth() * scale);
    output_format_.height = static_cast<int>(in_frame->GetHeight() * scale);
  }

  if (should_drop) {
    *out_frame = NULL;
    return true;
  }

  if (!StretchToOutputFrame(in_frame)) {
    return false;
  }

  *out_frame = output_frame_.get();
  return true;
}

bool VideoAdapter::StretchToOutputFrame(const VideoFrame* in_frame) {
  int output_width = output_format_.width;
  int output_height = output_format_.height;

  // Create and stretch the output frame if it has not been created yet or its
  // size is not same as the expected.
  bool stretched = false;
  if (!output_frame_.get() ||
      output_frame_->GetWidth() != static_cast<size_t>(output_width) ||
      output_frame_->GetHeight() != static_cast<size_t>(output_height)) {
    output_frame_.reset(
        in_frame->Stretch(output_width, output_height, true, true));
    if (!output_frame_.get()) {
      LOG(LS_WARNING) << "Adapter failed to stretch frame to "
                      << output_width << "x" << output_height;
      return false;
    }
    stretched = true;
    is_black_ = false;
  }

  if (!black_output_) {
    if (!stretched) {
      // The output frame does not need to be blacken and has not been stretched
      // from the input frame yet, stretch the input frame. This is the most
      // common case.
      in_frame->StretchToFrame(output_frame_.get(), true, true);
    }
    is_black_ = false;
  } else {
    if (!is_black_) {
      output_frame_->SetToBlack();
      is_black_ = true;
    }
    output_frame_->SetElapsedTime(in_frame->GetElapsedTime());
    output_frame_->SetTimeStamp(in_frame->GetTimeStamp());
  }

  return true;
}

///////////////////////////////////////////////////////////////////////
// Implementation of CoordinatedVideoAdapter
CoordinatedVideoAdapter::CoordinatedVideoAdapter()
    : cpu_adaptation_(false),
      gd_adaptation_(true),
      view_adaptation_(true),
      cpu_downgrade_count_(0),
      cpu_downgrade_wait_time_(0),
      view_desired_num_pixels_(INT_MAX),
      view_desired_interval_(0),
      encoder_desired_num_pixels_(INT_MAX),
      cpu_desired_num_pixels_(INT_MAX) {
}

// Helper function to UPGRADE or DOWNGRADE a number of pixels
void CoordinatedVideoAdapter::StepPixelCount(
    CoordinatedVideoAdapter::AdaptRequest request,
    int* num_pixels) {
  switch (request) {
    case CoordinatedVideoAdapter::DOWNGRADE:
      *num_pixels /= 2;
      break;

    case CoordinatedVideoAdapter::UPGRADE:
      *num_pixels *= 2;
      break;

    default:  // No change in pixel count
      break;
  }
  return;
}

// Find the adaptation request of the cpu based on the load. Return UPGRADE if
// the load is low, DOWNGRADE if the load is high, and KEEP otherwise.
CoordinatedVideoAdapter::AdaptRequest CoordinatedVideoAdapter::FindCpuRequest(
    int current_cpus, int max_cpus,
    float process_load, float system_load) {
  // Downgrade if system is high and plugin is at least more than midrange.
  if (system_load >= kHighSystemThreshold * max_cpus &&
      process_load >= kMediumProcessThreshold * current_cpus) {
    return CoordinatedVideoAdapter::DOWNGRADE;
  // Upgrade if system is low.
  } else if (system_load < kLowSystemThreshold * max_cpus) {
    return CoordinatedVideoAdapter::UPGRADE;
  }
  return CoordinatedVideoAdapter::KEEP;
}

// A remote view request for a new resolution.
void CoordinatedVideoAdapter::OnOutputFormatRequest(const VideoFormat& format) {
  talk_base::CritScope cs(&request_critical_section_);
  if (!view_adaptation_) {
    return;
  }
  // Set output for initial aspect ratio in mediachannel unittests.
  int old_num_pixels = GetOutputNumPixels();
  SetOutputFormat(format);
  SetOutputNumPixels(old_num_pixels);
  view_desired_num_pixels_ = format.width * format.height;
  view_desired_interval_ = format.interval;
  bool changed = AdaptToMinimumFormat();
  LOG(LS_INFO) << "VAdapt View Request: "
               << format.width << "x" << format.height
               << " Pixels: " << view_desired_num_pixels_
               << " Changed: " << (changed ? "true" : "false");
}

// A Bandwidth GD request for new resolution
void CoordinatedVideoAdapter::OnEncoderResolutionRequest(
    int width, int height, AdaptRequest request) {
  talk_base::CritScope cs(&request_critical_section_);
  if (!gd_adaptation_) {
    return;
  }
  if (KEEP != request) {
    int new_encoder_desired_num_pixels = width * height;
    int old_num_pixels = GetOutputNumPixels();
    if (new_encoder_desired_num_pixels != old_num_pixels) {
      LOG(LS_VERBOSE) << "VAdapt GD resolution stale.  Ignored";
    } else {
      // Update the encoder desired format based on the request.
      encoder_desired_num_pixels_ = new_encoder_desired_num_pixels;
      StepPixelCount(request, &encoder_desired_num_pixels_);
    }
  }
  bool changed = AdaptToMinimumFormat();
  LOG(LS_INFO) << "VAdapt GD Request: "
               << (DOWNGRADE == request ? "down" :
                   (UPGRADE == request ? "up" : "keep"))
               << " From: " << width << "x" << height
               << " Pixels: " << encoder_desired_num_pixels_
               << " Changed: " << (changed ? "true" : "false");
}

// A CPU request for new resolution
void CoordinatedVideoAdapter::OnCpuLoadUpdated(
    int current_cpus, int max_cpus, float process_load, float system_load) {
  talk_base::CritScope cs(&request_critical_section_);
  if (!cpu_adaptation_) {
    return;
  }
  AdaptRequest request = FindCpuRequest(current_cpus, max_cpus,
                                        process_load, system_load);
  // Update how many times we have downgraded due to the cpu load.
  switch (request) {
    case DOWNGRADE:
      if (cpu_downgrade_count_ < kMaxCpuDowngrades) {
        // Ignore downgrades if we have downgraded the maximum times or we just
        // downgraded in a short time.
        if (cpu_downgrade_wait_time_ != 0 &&
            talk_base::TimeIsLater(talk_base::Time(),
                                   cpu_downgrade_wait_time_)) {
          LOG(LS_VERBOSE) << "VAdapt CPU load high but do not downgrade until "
                          << talk_base::TimeUntil(cpu_downgrade_wait_time_)
                          << " ms.";
          request = KEEP;
        } else {
          ++cpu_downgrade_count_;
        }
      } else {
          LOG(LS_VERBOSE) << "VAdapt CPU load high but do not downgrade "
                             "because maximum downgrades reached";
      }
      break;
    case UPGRADE:
      if (cpu_downgrade_count_ > 0) {
        bool is_min = IsMinimumFormat(cpu_desired_num_pixels_);
        if (is_min) {
          --cpu_downgrade_count_;
        } else {
         LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade "
                             "because cpu is not limiting resolution";
        }
      } else {
          LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade "
                             "because minimum downgrades reached";
      }
      break;
    case KEEP:
    default:
      break;
  }
  if (KEEP != request) {
    // TODO: compute stepping up/down from OutputNumPixels but
    // clamp to inputpixels / 4 (2 steps)
    cpu_desired_num_pixels_ = static_cast<int>(
        input_format().width * input_format().height >> cpu_downgrade_count_);
  }
  bool changed = AdaptToMinimumFormat();
  LOG(LS_INFO) << "VAdapt CPU Request: "
               << (DOWNGRADE == request ? "down" :
                   (UPGRADE == request ? "up" : "keep"))
               << " Process: " << process_load
               << " System: " << system_load
               << " Steps: " << cpu_downgrade_count_
               << " Changed: " << (changed ? "true" : "false");
}

// Called by cpu adapter on up requests.
bool CoordinatedVideoAdapter::IsMinimumFormat(int pixels) {
  // Find closest scale factor that matches input resolution to min_num_pixels
  // and set that for output resolution.  This is not needed for VideoAdapter,
  // but provides feedback to unittests and users on expected resolution.
  // Actual resolution is based on input frame.
  VideoFormat new_output = output_format();
  VideoFormat input = input_format();
  if (input_format().IsSize0x0()) {
    input = new_output;
  }
  float scale = 1.0f;
  if (!input.IsSize0x0()) {
    scale = FindClosestScale(input.width,
                             input.height,
                             pixels);
  }
  new_output.width = static_cast<int>(input.width * scale);
  new_output.height = static_cast<int>(input.height * scale);
  int new_pixels = new_output.width * new_output.height;
  int num_pixels = GetOutputNumPixels();
  return new_pixels <= num_pixels;
}

// Called by all coordinators when there is a change.
bool CoordinatedVideoAdapter::AdaptToMinimumFormat() {
  int old_num_pixels = GetOutputNumPixels();
  // Get the min of the formats that the server, encoder, and cpu wants.
  int min_num_pixels = view_desired_num_pixels_;
  if (encoder_desired_num_pixels_ &&
      (encoder_desired_num_pixels_ < min_num_pixels)) {
    min_num_pixels = encoder_desired_num_pixels_;
  }
  if (cpu_adaptation_ && cpu_desired_num_pixels_ &&
      (cpu_desired_num_pixels_ < min_num_pixels)) {
    min_num_pixels = cpu_desired_num_pixels_;
    // Update the cpu_downgrade_wait_time_ if we are going to downgrade video.
    cpu_downgrade_wait_time_ =
      talk_base::TimeAfter(kDefaultDowngradeWaitTimeMs);
  }
  // prevent going below QQVGA
  if (min_num_pixels > 0 && min_num_pixels < kMinNumPixels) {
    min_num_pixels = kMinNumPixels;
  }
  SetOutputNumPixels(min_num_pixels);

  // Find closest scale factor that matches input resolution to min_num_pixels
  // and set that for output resolution.  This is not needed for VideoAdapter,
  // but provides feedback to unittests and users on expected resolution.
  // Actual resolution is based on input frame.
  VideoFormat new_output = output_format();
  VideoFormat input = input_format();
  if (input_format().IsSize0x0()) {
    input = new_output;
  }
  float scale = 1.0f;
  if (!input.IsSize0x0()) {
    scale = FindClosestScale(input.width,
                             input.height,
                             min_num_pixels);
  }
  new_output.width = static_cast<int>(input.width * scale);
  new_output.height = static_cast<int>(input.height * scale);
  new_output.interval = view_desired_interval_;
  SetOutputFormat(new_output);
  int new_num_pixels = GetOutputNumPixels();
  bool changed = new_num_pixels != old_num_pixels;

  LOG(LS_VERBOSE) << "VAdapt Status View: " << view_desired_num_pixels_
                  << " GD: " << encoder_desired_num_pixels_
                  << " CPU: " << cpu_desired_num_pixels_
                  << " Pixels: " << min_num_pixels
                  << " Scale: " << scale
                  << " Resolution: " << new_output.width
                  << "x" << new_output.height
                  << " Changed: " << (changed ? "true" : "false");
  return changed;
}

}  // namespace cricket