Index: webrtc/modules/remote_bitrate_estimator/test/metric_recorder.cc |
diff --git a/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.cc b/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.cc |
new file mode 100644 |
index 0000000000000000000000000000000000000000..c9033c564f473698d97d2344426357fefc5bb3b3 |
--- /dev/null |
+++ b/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.cc |
@@ -0,0 +1,438 @@ |
+/* |
+ * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. |
+ * |
+ * Use of this source code is governed by a BSD-style license |
+ * that can be found in the LICENSE file in the root of the source |
+ * tree. An additional intellectual property rights grant can be found |
+ * in the file PATENTS. All contributing project authors may |
+ * be found in the AUTHORS file in the root of the source tree. |
+ */ |
+ |
+#include "webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h" |
+ |
+#include <algorithm> |
+ |
+namespace webrtc { |
+namespace testing { |
+namespace bwe { |
+ |
+namespace { |
+ |
+template <typename T> |
+T Sum(const std::vector<T>& input) { |
+ T total = 0; |
+ for (T val : input) { |
+ total += val; |
+ } |
+ return total; |
+} |
+ |
+template <typename T> |
+double Average(const std::vector<T>& array, size_t size) { |
+ return static_cast<double>(Sum(array)) / size; |
+} |
+ |
+template <typename T> |
+std::vector<T> Abs(const std::vector<T>& input) { |
+ std::vector<T> output(input); |
+ for (T val : output) { |
+ val = std::abs(val); |
+ } |
+ return output; |
+} |
+ |
+template <typename T> |
+std::vector<double> Pow(const std::vector<T>& input, double p) { |
+ std::vector<double> output; |
+ for (T val : input) { |
+ output.push_back(pow(static_cast<double>(val), p)); |
+ } |
+ return output; |
+} |
+ |
+template <typename T> |
+double StandardDeviation(const std::vector<T>& array, size_t size) { |
+ double mean = Average(array, size); |
+ std::vector<double> square_values = Pow(array, 2.0); |
+ double var = Average(square_values, size) - mean * mean; |
+ return sqrt(var); |
+} |
+ |
+// Holder mean, Manhattan distance for p=1, EuclidianNorm/sqrt(n) for p=2. |
+template <typename T> |
+double NormLp(const std::vector<T>& array, size_t size, double p) { |
+ std::vector<T> abs_values = Abs(array); |
+ std::vector<double> pow_values = Pow(abs_values, p); |
+ return pow(Sum(pow_values) / size, 1.0 / p); |
+} |
+ |
+template <typename T> |
+std::vector<T> PositiveFilter(const std::vector<T>& input) { |
+ std::vector<T> output(input); |
+ for (T val : output) { |
+ val = val > 0 ? val : 0; |
+ } |
+ return output; |
+} |
+ |
+template <typename T> |
+std::vector<T> NegativeFilter(const std::vector<T>& input) { |
+ std::vector<T> output(input); |
+ for (T val : output) { |
+ val = val < 0 ? -val : 0; |
+ } |
+ return output; |
+} |
+} // namespace |
+ |
+LinkShare::LinkShare(ChokeFilter* choke_filter) |
+ : choke_filter_(choke_filter), running_flows_(choke_filter->flow_ids()) { |
+} |
+ |
+void LinkShare::PauseFlow(int flow_id) { |
+ running_flows_.erase(flow_id); |
+} |
+ |
+void LinkShare::ResumeFlow(int flow_id) { |
+ running_flows_.insert(flow_id); |
+} |
+ |
+uint32_t LinkShare::TotalAvailableKbps() { |
+ return choke_filter_->capacity_kbps(); |
+} |
+ |
+uint32_t LinkShare::AvailablePerFlowKbps(int flow_id) { |
+ uint32_t available_capacity_per_flow_kbps = 0; |
+ if (running_flows_.find(flow_id) != running_flows_.end()) { |
+ available_capacity_per_flow_kbps = |
+ TotalAvailableKbps() / static_cast<uint32_t>(running_flows_.size()); |
+ } |
+ return available_capacity_per_flow_kbps; |
+} |
+ |
+MetricRecorder::MetricRecorder(const std::string algorithm_name, |
+ int flow_id, |
+ PacketSender* packet_sender, |
+ LinkShare* link_share) |
+ : algorithm_name_(algorithm_name), |
+ flow_id_(flow_id), |
+ packet_sender_(packet_sender), |
+ link_share_(link_share), |
+ now_ms_(0), |
+ delays_ms_(), |
+ throughput_bytes_(), |
+ weighted_estimate_error_(), |
+ last_unweighted_estimate_error_(0), |
+ optimal_throughput_bits_(0), |
+ last_available_bitrate_per_flow_kbps_(0), |
+ start_computing_metrics_ms_(0), |
+ started_computing_metrics_(false) { |
+} |
+ |
+void MetricRecorder::SetPlotInformation( |
+ const std::vector<std::string>& prefixes) { |
+ assert(prefixes.size() == kNumMetrics); |
+ for (size_t i = 0; i < kNumMetrics; ++i) { |
+ plot_information_[i].prefix = prefixes[i]; |
+ } |
+ plot_information_[kThroughput].plot_interval_ms = 100; |
+ plot_information_[kDelay].plot_interval_ms = 100; |
+ plot_information_[kLoss].plot_interval_ms = 500; |
+ plot_information_[kObjective].plot_interval_ms = 1000; |
+ plot_information_[kTotalAvailable].plot_interval_ms = 1000; |
+ plot_information_[kAvailablePerFlow].plot_interval_ms = 1000; |
+ |
+ for (int i = kThroughput; i < kNumMetrics; ++i) { |
+ plot_information_[i].last_plot_ms = 0; |
+ if (i == kObjective || i == kAvailablePerFlow) { |
+ plot_information_[i].plot = false; |
+ } else { |
+ plot_information_[i].plot = true; |
+ } |
+ } |
+} |
+ |
+void MetricRecorder::PlotAllDynamics() { |
+ for (int i = kThroughput; i < kNumMetrics; ++i) { |
+ if (plot_information_[i].plot && |
+ now_ms_ - plot_information_[i].last_plot_ms >= |
+ plot_information_[i].plot_interval_ms) { |
+ PlotDynamics(i); |
+ } |
+ } |
+} |
+ |
+void MetricRecorder::PlotDynamics(int metric) { |
+ if (metric == kTotalAvailable) { |
+ BWE_TEST_LOGGING_PLOT_WITH_NAME( |
+ 0, plot_information_[kTotalAvailable].prefix, now_ms_, |
+ GetTotalAvailableKbps(), "Available"); |
+ } else if (metric == kAvailablePerFlow) { |
+ BWE_TEST_LOGGING_PLOT_WITH_NAME( |
+ 0, plot_information_[kAvailablePerFlow].prefix, now_ms_, |
+ GetAvailablePerFlowKbps(), "Available_per_flow"); |
+ } else { |
+ PlotLine(metric, plot_information_[metric].prefix, |
+ plot_information_[metric].time_ms, |
+ plot_information_[metric].value); |
+ } |
+ plot_information_[metric].last_plot_ms = now_ms_; |
+} |
+ |
+template <typename T> |
+void MetricRecorder::PlotLine(int windows_id, |
+ const std::string& prefix, |
+ int64_t time_ms, |
+ T y) { |
+ BWE_TEST_LOGGING_PLOT_WITH_NAME(windows_id, prefix, time_ms, |
+ static_cast<double>(y), algorithm_name_); |
+} |
+ |
+void MetricRecorder::UpdateTime(int64_t time_ms) { |
+ now_ms_ = std::max(now_ms_, time_ms); |
+} |
+ |
+void MetricRecorder::UpdateThroughput(int64_t bitrate_kbps, |
+ size_t payload_size) { |
+ // Total throughput should be computed before updating the time. |
+ PushThroughputBytes(payload_size, now_ms_); |
+ plot_information_[kThroughput].Update(now_ms_, bitrate_kbps); |
+} |
+ |
+void MetricRecorder::UpdateDelay(int64_t delay_ms) { |
+ PushDelayMs(delay_ms, now_ms_); |
+ plot_information_[kDelay].Update(now_ms_, delay_ms); |
+} |
+ |
+void MetricRecorder::UpdateLoss(float loss_ratio) { |
+ plot_information_[kLoss].Update(now_ms_, loss_ratio); |
+} |
+ |
+void MetricRecorder::UpdateObjective() { |
+ plot_information_[kObjective].Update(now_ms_, ObjectiveFunction()); |
+} |
+ |
+uint32_t MetricRecorder::GetTotalAvailableKbps() { |
+ return link_share_->TotalAvailableKbps(); |
+} |
+ |
+uint32_t MetricRecorder::GetAvailablePerFlowKbps() { |
+ return link_share_->AvailablePerFlowKbps(flow_id_); |
+} |
+ |
+uint32_t MetricRecorder::GetSendingEstimateKbps() { |
+ return packet_sender_->TargetBitrateKbps(); |
+} |
+ |
+void MetricRecorder::PushDelayMs(int64_t delay_ms, int64_t arrival_time_ms) { |
+ if (ShouldRecord(arrival_time_ms)) { |
+ delays_ms_.push_back(delay_ms); |
+ } |
+} |
+ |
+void MetricRecorder::PushThroughputBytes(size_t payload_size, |
+ int64_t arrival_time_ms) { |
+ if (ShouldRecord(arrival_time_ms)) { |
+ throughput_bytes_.push_back(payload_size); |
+ |
+ int64_t current_available_per_flow_kbps = |
+ static_cast<int64_t>(GetAvailablePerFlowKbps()); |
+ |
+ int64_t current_bitrate_diff_kbps = |
+ static_cast<int64_t>(GetSendingEstimateKbps()) - |
+ current_available_per_flow_kbps; |
+ |
+ weighted_estimate_error_.push_back( |
+ ((current_bitrate_diff_kbps + last_unweighted_estimate_error_) * |
+ (arrival_time_ms - plot_information_[kThroughput].time_ms)) / |
+ 2); |
+ |
+ optimal_throughput_bits_ += |
+ ((current_available_per_flow_kbps + |
+ last_available_bitrate_per_flow_kbps_) * |
+ (arrival_time_ms - plot_information_[kThroughput].time_ms)) / |
+ 2; |
+ |
+ last_available_bitrate_per_flow_kbps_ = current_available_per_flow_kbps; |
+ } |
+} |
+ |
+bool MetricRecorder::ShouldRecord(int64_t arrival_time_ms) { |
+ if (arrival_time_ms >= start_computing_metrics_ms_) { |
+ if (!started_computing_metrics_) { |
+ start_computing_metrics_ms_ = arrival_time_ms; |
+ now_ms_ = arrival_time_ms; |
+ started_computing_metrics_ = true; |
+ } |
+ return true; |
+ } else { |
+ return false; |
+ } |
+} |
+ |
+// The weighted_estimate_error_ was weighted based on time windows. |
+// This function scales back the result before plotting. |
+double MetricRecorder::Renormalize(double x) { |
+ size_t num_packets_received = delays_ms_.size(); |
+ return (x * num_packets_received) / now_ms_; |
+} |
+ |
+inline double U(int64_t x, double alpha) { |
+ if (alpha == 1.0) { |
+ return log(static_cast<double>(x)); |
+ } |
+ return pow(static_cast<double>(x), 1.0 - alpha) / (1.0 - alpha); |
+} |
+ |
+inline double U(size_t x, double alpha) { |
+ return U(static_cast<int64_t>(x), alpha); |
+} |
+ |
+// TODO(magalhaesc): Update ObjectiveFunction. |
+double MetricRecorder::ObjectiveFunction() { |
+ const double kDelta = 0.15; // Delay penalty factor. |
+ const double kAlpha = 1.0; |
+ const double kBeta = 1.0; |
+ |
+ double throughput_metric = U(Sum(throughput_bytes_), kAlpha); |
+ double delay_penalty = kDelta * U(Sum(delays_ms_), kBeta); |
+ |
+ return throughput_metric - delay_penalty; |
+} |
+ |
+void MetricRecorder::PlotThroughputHistogram(const std::string& title, |
+ const std::string& bwe_name, |
+ int num_flows, |
+ int64_t extra_offset_ms, |
+ const std::string optimum_id) { |
+ size_t num_packets_received = delays_ms_.size(); |
+ |
+ int64_t duration_ms = now_ms_ - start_computing_metrics_ms_ - extra_offset_ms; |
+ |
+ double average_bitrate_kbps = |
+ static_cast<double>(8 * Sum(throughput_bytes_) / duration_ms); |
+ |
+ double optimal_bitrate_per_flow_kbps = |
+ static_cast<double>(optimal_throughput_bits_ / duration_ms); |
+ |
+ std::vector<int64_t> positive = PositiveFilter(weighted_estimate_error_); |
+ std::vector<int64_t> negative = NegativeFilter(weighted_estimate_error_); |
+ |
+ double p_error = Renormalize(NormLp(positive, num_packets_received, 1.0)); |
+ double n_error = Renormalize(NormLp(negative, num_packets_received, 1.0)); |
+ |
+ // Prevent the error to be too close to zero (plotting issue). |
+ double extra_error = average_bitrate_kbps / 500; |
+ |
+ std::string optimum_title = |
+ optimum_id.empty() ? "optimal_bitrate" : "optimal_bitrates#" + optimum_id; |
+ |
+ BWE_TEST_LOGGING_LABEL(4, title, "average_bitrate_(kbps)", num_flows); |
+ BWE_TEST_LOGGING_LIMITERRORBAR( |
+ 4, bwe_name, average_bitrate_kbps, |
+ average_bitrate_kbps - n_error - extra_error, |
+ average_bitrate_kbps + p_error + extra_error, "estimate_error", |
+ optimal_bitrate_per_flow_kbps, optimum_title, flow_id_); |
+ |
+ BWE_TEST_LOGGING_LOG1("RESULTS >>> " + bwe_name + " Channel utilization : ", |
+ "%lf %%", |
+ 100.0 * static_cast<double>(average_bitrate_kbps) / |
+ optimal_bitrate_per_flow_kbps); |
+ |
+ RTC_UNUSED(p_error); |
+ RTC_UNUSED(n_error); |
+ RTC_UNUSED(extra_error); |
+ RTC_UNUSED(optimal_bitrate_per_flow_kbps); |
+} |
+ |
+void MetricRecorder::PlotThroughputHistogram(const std::string& title, |
+ const std::string& bwe_name, |
+ int num_flows, |
+ int64_t extra_offset_ms) { |
+ PlotThroughputHistogram(title, bwe_name, num_flows, extra_offset_ms, ""); |
+} |
+ |
+void MetricRecorder::PlotDelayHistogram(const std::string& title, |
+ const std::string& bwe_name, |
+ int num_flows, |
+ int64_t one_way_path_delay_ms) { |
+ size_t num_packets_received = delays_ms_.size(); |
+ double average_delay_ms = Average(delays_ms_, num_packets_received); |
+ |
+ // Prevent the error to be too close to zero (plotting issue). |
+ double extra_error = average_delay_ms / 500; |
+ |
+ double tenth_sigma_ms = |
+ StandardDeviation(delays_ms_, num_packets_received) / 10.0 + extra_error; |
+ |
+ size_t per_5_index = (num_packets_received - 1) / 20; |
+ std::nth_element(delays_ms_.begin(), delays_ms_.begin() + per_5_index, |
+ delays_ms_.end()); |
+ int64_t percentile_5_ms = delays_ms_[per_5_index]; |
+ |
+ size_t per_95_index = num_packets_received - 1 - per_5_index; |
+ std::nth_element(delays_ms_.begin(), delays_ms_.begin() + per_95_index, |
+ delays_ms_.end()); |
+ int64_t percentile_95_ms = delays_ms_[per_95_index]; |
+ |
+ BWE_TEST_LOGGING_LABEL(5, title, "average_delay_(ms)", num_flows) |
+ BWE_TEST_LOGGING_ERRORBAR(5, bwe_name, average_delay_ms, percentile_5_ms, |
+ percentile_95_ms, "5th and 95th percentiles", |
+ flow_id_); |
+ |
+ // Log added latency, disregard baseline path delay. |
+ BWE_TEST_LOGGING_LOG1("RESULTS >>> " + bwe_name + " Delay average : ", |
+ "%lf ms", average_delay_ms - one_way_path_delay_ms); |
+ BWE_TEST_LOGGING_LOG1("RESULTS >>> " + bwe_name + " Delay 5th percentile : ", |
+ "%ld ms", percentile_5_ms - one_way_path_delay_ms); |
+ BWE_TEST_LOGGING_LOG1("RESULTS >>> " + bwe_name + " Delay 95th percentile : ", |
+ "%ld ms", percentile_95_ms - one_way_path_delay_ms); |
+ |
+ RTC_UNUSED(tenth_sigma_ms); |
+ RTC_UNUSED(percentile_5_ms); |
+ RTC_UNUSED(percentile_95_ms); |
+} |
+ |
+void MetricRecorder::PlotLossHistogram(const std::string& title, |
+ const std::string& bwe_name, |
+ int num_flows, |
+ float global_loss_ratio) { |
+ BWE_TEST_LOGGING_LABEL(6, title, "packet_loss_ratio_(%)", num_flows) |
+ BWE_TEST_LOGGING_BAR(6, bwe_name, 100.0f * global_loss_ratio, flow_id_); |
+ |
+ BWE_TEST_LOGGING_LOG1("RESULTS >>> " + bwe_name + " Loss Ratio : ", "%f %%", |
+ 100.0f * global_loss_ratio); |
+} |
+ |
+void MetricRecorder::PlotObjectiveHistogram(const std::string& title, |
+ const std::string& bwe_name, |
+ int num_flows) { |
+ BWE_TEST_LOGGING_LABEL(7, title, "objective_function", num_flows) |
+ BWE_TEST_LOGGING_BAR(7, bwe_name, ObjectiveFunction(), flow_id_); |
+} |
+ |
+void MetricRecorder::PlotZero() { |
+ for (int i = kThroughput; i <= kLoss; ++i) { |
+ if (plot_information_[i].plot) { |
+ std::stringstream prefix; |
+ prefix << "Receiver_" << flow_id_ << "_" + plot_information_[i].prefix; |
+ PlotLine(i, prefix.str(), now_ms_, 0); |
+ plot_information_[i].last_plot_ms = now_ms_; |
+ } |
+ } |
+} |
+ |
+void MetricRecorder::PauseFlow() { |
+ PlotZero(); |
+ link_share_->PauseFlow(flow_id_); |
+} |
+ |
+void MetricRecorder::ResumeFlow(int64_t paused_time_ms) { |
+ UpdateTime(now_ms_ + paused_time_ms); |
+ PlotZero(); |
+ link_share_->ResumeFlow(flow_id_); |
+} |
+ |
+} // namespace bwe |
+} // namespace testing |
+} // namespace webrtc |