1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/remote_bitrate_estimator/test/bwe_test.h"
12
13 #include <memory>
14 #include <sstream>
15
16 #include "modules/include/module_common_types.h"
17 #include "modules/remote_bitrate_estimator/test/bwe_test_framework.h"
18 #include "modules/remote_bitrate_estimator/test/metric_recorder.h"
19 #include "modules/remote_bitrate_estimator/test/packet_receiver.h"
20 #include "modules/remote_bitrate_estimator/test/packet_sender.h"
21 #include "rtc_base/arraysize.h"
22 #include "system_wrappers/include/clock.h"
23 #include "system_wrappers/include/field_trial.h"
24 #include "test/testsupport/perf_test.h"
25
26 using std::vector;
27
28 namespace {
29 const int kQuickTestTimeoutMs = 500;
30 }
31
32 namespace webrtc {
33 namespace testing {
34 namespace bwe {
35
PacketProcessorRunner(PacketProcessor * processor)36 PacketProcessorRunner::PacketProcessorRunner(PacketProcessor* processor)
37 : processor_(processor) {
38 }
39
~PacketProcessorRunner()40 PacketProcessorRunner::~PacketProcessorRunner() {
41 for (Packet* packet : queue_)
42 delete packet;
43 }
44
RunsProcessor(const PacketProcessor * processor) const45 bool PacketProcessorRunner::RunsProcessor(
46 const PacketProcessor* processor) const {
47 return processor == processor_;
48 }
49
RunFor(int64_t time_ms,int64_t time_now_ms,Packets * in_out)50 void PacketProcessorRunner::RunFor(int64_t time_ms,
51 int64_t time_now_ms,
52 Packets* in_out) {
53 Packets to_process;
54 FindPacketsToProcess(processor_->flow_ids(), in_out, &to_process);
55 processor_->RunFor(time_ms, &to_process);
56 QueuePackets(&to_process, time_now_ms * 1000);
57 if (!to_process.empty()) {
58 processor_->Plot(to_process.back()->send_time_ms());
59 }
60 in_out->merge(to_process, DereferencingComparator<Packet>);
61 }
62
FindPacketsToProcess(const FlowIds & flow_ids,Packets * in,Packets * out)63 void PacketProcessorRunner::FindPacketsToProcess(const FlowIds& flow_ids,
64 Packets* in,
65 Packets* out) {
66 assert(out->empty());
67 for (Packets::iterator it = in->begin(); it != in->end();) {
68 // TODO(holmer): Further optimize this by looking for consecutive flow ids
69 // in the packet list and only doing the binary search + splice once for a
70 // sequence.
71 if (flow_ids.find((*it)->flow_id()) != flow_ids.end()) {
72 Packets::iterator next = it;
73 ++next;
74 out->splice(out->end(), *in, it);
75 it = next;
76 } else {
77 ++it;
78 }
79 }
80 }
81
QueuePackets(Packets * batch,int64_t end_of_batch_time_us)82 void PacketProcessorRunner::QueuePackets(Packets* batch,
83 int64_t end_of_batch_time_us) {
84 queue_.merge(*batch, DereferencingComparator<Packet>);
85 if (queue_.empty()) {
86 return;
87 }
88 Packets::iterator it = queue_.begin();
89 for (; it != queue_.end(); ++it) {
90 if ((*it)->send_time_us() > end_of_batch_time_us) {
91 break;
92 }
93 }
94 Packets to_transfer;
95 to_transfer.splice(to_transfer.begin(), queue_, queue_.begin(), it);
96 batch->merge(to_transfer, DereferencingComparator<Packet>);
97 }
98
99 // Plot link capacity by default.
BweTest()100 BweTest::BweTest() : BweTest(true) {
101 }
102
BweTest(bool plot_capacity)103 BweTest::BweTest(bool plot_capacity)
104 : run_time_ms_(0),
105 time_now_ms_(-1),
106 simulation_interval_ms_(-1),
107 plot_total_available_capacity_(plot_capacity) {
108 links_.push_back(&uplink_);
109 links_.push_back(&downlink_);
110 }
111
~BweTest()112 BweTest::~BweTest() {
113 for (Packet* packet : packets_)
114 delete packet;
115 }
116
SetUp()117 void BweTest::SetUp() {
118 const ::testing::TestInfo* const test_info =
119 ::testing::UnitTest::GetInstance()->current_test_info();
120 std::string test_name =
121 std::string(test_info->test_case_name()) + "_" +
122 std::string(test_info->name());
123 BWE_TEST_LOGGING_GLOBAL_CONTEXT(test_name);
124 BWE_TEST_LOGGING_GLOBAL_ENABLE(false);
125 }
126
AddPacketProcessor(PacketProcessor * processor,ProcessorType processor_type)127 void Link::AddPacketProcessor(PacketProcessor* processor,
128 ProcessorType processor_type) {
129 assert(processor);
130 switch (processor_type) {
131 case kSender:
132 senders_.push_back(static_cast<PacketSender*>(processor));
133 break;
134 case kReceiver:
135 receivers_.push_back(static_cast<PacketReceiver*>(processor));
136 break;
137 case kRegular:
138 break;
139 }
140 processors_.push_back(PacketProcessorRunner(processor));
141 }
142
RemovePacketProcessor(PacketProcessor * processor)143 void Link::RemovePacketProcessor(PacketProcessor* processor) {
144 for (std::vector<PacketProcessorRunner>::iterator it = processors_.begin();
145 it != processors_.end(); ++it) {
146 if (it->RunsProcessor(processor)) {
147 processors_.erase(it);
148 return;
149 }
150 }
151 assert(false);
152 }
153
154 // Ownership of the created packets is handed over to the caller.
Run(int64_t run_for_ms,int64_t now_ms,Packets * packets)155 void Link::Run(int64_t run_for_ms, int64_t now_ms, Packets* packets) {
156 for (auto& processor : processors_) {
157 processor.RunFor(run_for_ms, now_ms, packets);
158 }
159 }
160
VerboseLogging(bool enable)161 void BweTest::VerboseLogging(bool enable) {
162 BWE_TEST_LOGGING_GLOBAL_ENABLE(enable);
163 }
164
RunFor(int64_t time_ms)165 void BweTest::RunFor(int64_t time_ms) {
166 // Set simulation interval from first packet sender.
167 // TODO(holmer): Support different feedback intervals for different flows.
168
169 // For quick perf tests ignore passed timeout
170 if (field_trial::IsEnabled("WebRTC-QuickPerfTest")) {
171 time_ms = kQuickTestTimeoutMs;
172 }
173 if (!uplink_.senders().empty()) {
174 simulation_interval_ms_ = uplink_.senders()[0]->GetFeedbackIntervalMs();
175 } else if (!downlink_.senders().empty()) {
176 simulation_interval_ms_ = downlink_.senders()[0]->GetFeedbackIntervalMs();
177 }
178 assert(simulation_interval_ms_ > 0);
179 if (time_now_ms_ == -1) {
180 time_now_ms_ = simulation_interval_ms_;
181 }
182 for (run_time_ms_ += time_ms;
183 time_now_ms_ <= run_time_ms_ - simulation_interval_ms_;
184 time_now_ms_ += simulation_interval_ms_) {
185 // Packets are first generated on the first link, passed through all the
186 // PacketProcessors and PacketReceivers. The PacketReceivers produces
187 // FeedbackPackets which are then processed by the next link, where they
188 // at some point will be consumed by a PacketSender.
189 for (Link* link : links_)
190 link->Run(simulation_interval_ms_, time_now_ms_, &packets_);
191 }
192 }
193
GetTestName() const194 std::string BweTest::GetTestName() const {
195 const ::testing::TestInfo* const test_info =
196 ::testing::UnitTest::GetInstance()->current_test_info();
197 return std::string(test_info->name());
198 }
199
PrintResults(double max_throughput_kbps,Stats<double> throughput_kbps,int flow_id,Stats<double> flow_delay_ms,Stats<double> flow_throughput_kbps)200 void BweTest::PrintResults(double max_throughput_kbps,
201 Stats<double> throughput_kbps,
202 int flow_id,
203 Stats<double> flow_delay_ms,
204 Stats<double> flow_throughput_kbps) {
205 std::map<int, Stats<double>> flow_delays_ms;
206 flow_delays_ms[flow_id] = flow_delay_ms;
207 std::map<int, Stats<double>> flow_throughputs_kbps;
208 flow_throughputs_kbps[flow_id] = flow_throughput_kbps;
209 PrintResults(max_throughput_kbps, throughput_kbps, flow_delays_ms,
210 flow_throughputs_kbps);
211 }
212
PrintResults(double max_throughput_kbps,Stats<double> throughput_kbps,std::map<int,Stats<double>> flow_delay_ms,std::map<int,Stats<double>> flow_throughput_kbps)213 void BweTest::PrintResults(double max_throughput_kbps,
214 Stats<double> throughput_kbps,
215 std::map<int, Stats<double>> flow_delay_ms,
216 std::map<int, Stats<double>> flow_throughput_kbps) {
217 double utilization = throughput_kbps.GetMean() / max_throughput_kbps;
218 webrtc::test::PrintResult("BwePerformance", GetTestName(), "Utilization",
219 utilization * 100.0, "%", false);
220 webrtc::test::PrintResult(
221 "BwePerformance", GetTestName(), "Utilization var coeff",
222 throughput_kbps.GetStdDev() / throughput_kbps.GetMean(), "", false);
223 std::stringstream ss;
224 for (auto& kv : flow_throughput_kbps) {
225 ss.str("");
226 ss << "Throughput flow " << kv.first;
227 webrtc::test::PrintResultMeanAndError("BwePerformance", GetTestName(),
228 ss.str(), kv.second.GetMean(),
229 kv.second.GetStdDev(), "kbps", false);
230 }
231 for (auto& kv : flow_delay_ms) {
232 ss.str("");
233 ss << "Delay flow " << kv.first;
234 webrtc::test::PrintResultMeanAndError("BwePerformance", GetTestName(),
235 ss.str(), kv.second.GetMean(),
236 kv.second.GetStdDev(), "ms", false);
237 }
238 double fairness_index = 1.0;
239 if (!flow_throughput_kbps.empty()) {
240 double squared_bitrate_sum = 0.0;
241 fairness_index = 0.0;
242 for (auto kv : flow_throughput_kbps) {
243 squared_bitrate_sum += kv.second.GetMean() * kv.second.GetMean();
244 fairness_index += kv.second.GetMean();
245 }
246 fairness_index *= fairness_index;
247 fairness_index /= flow_throughput_kbps.size() * squared_bitrate_sum;
248 }
249 webrtc::test::PrintResult("BwePerformance", GetTestName(), "Fairness",
250 fairness_index * 100, "%", false);
251 }
252
RunFairnessTest(BandwidthEstimatorType bwe_type,size_t num_media_flows,size_t num_tcp_flows,int64_t run_time_seconds,uint32_t capacity_kbps,int64_t max_delay_ms,int64_t rtt_ms,int64_t max_jitter_ms,const int64_t * offsets_ms)253 void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type,
254 size_t num_media_flows,
255 size_t num_tcp_flows,
256 int64_t run_time_seconds,
257 uint32_t capacity_kbps,
258 int64_t max_delay_ms,
259 int64_t rtt_ms,
260 int64_t max_jitter_ms,
261 const int64_t* offsets_ms) {
262 RunFairnessTest(bwe_type, num_media_flows, num_tcp_flows, run_time_seconds,
263 capacity_kbps, max_delay_ms, rtt_ms, max_jitter_ms,
264 offsets_ms, "Fairness_test", bwe_names[bwe_type]);
265 }
266
RunFairnessTest(BandwidthEstimatorType bwe_type,size_t num_media_flows,size_t num_tcp_flows,int64_t run_time_seconds,uint32_t capacity_kbps,int64_t max_delay_ms,int64_t rtt_ms,int64_t max_jitter_ms,const int64_t * offsets_ms,const std::string & title,const std::string & flow_name)267 void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type,
268 size_t num_media_flows,
269 size_t num_tcp_flows,
270 int64_t run_time_seconds,
271 uint32_t capacity_kbps,
272 int64_t max_delay_ms,
273 int64_t rtt_ms,
274 int64_t max_jitter_ms,
275 const int64_t* offsets_ms,
276 const std::string& title,
277 const std::string& flow_name) {
278 std::set<int> all_flow_ids;
279 std::set<int> media_flow_ids;
280 std::set<int> tcp_flow_ids;
281 int next_flow_id = 0;
282 for (size_t i = 0; i < num_media_flows; ++i) {
283 media_flow_ids.insert(next_flow_id);
284 all_flow_ids.insert(next_flow_id);
285 ++next_flow_id;
286 }
287 for (size_t i = 0; i < num_tcp_flows; ++i) {
288 tcp_flow_ids.insert(next_flow_id);
289 all_flow_ids.insert(next_flow_id);
290 ++next_flow_id;
291 }
292
293 std::vector<VideoSource*> sources;
294 std::vector<PacketSender*> senders;
295 std::vector<MetricRecorder*> metric_recorders;
296
297 int64_t max_offset_ms = 0;
298
299 for (int media_flow : media_flow_ids) {
300 sources.push_back(new AdaptiveVideoSource(media_flow, 30, 300, 0,
301 offsets_ms[media_flow]));
302 senders.push_back(new PacedVideoSender(&uplink_, sources.back(), bwe_type));
303 max_offset_ms = std::max(max_offset_ms, offsets_ms[media_flow]);
304 }
305
306 for (int tcp_flow : tcp_flow_ids) {
307 senders.push_back(new TcpSender(&uplink_, tcp_flow, offsets_ms[tcp_flow]));
308 max_offset_ms = std::max(max_offset_ms, offsets_ms[tcp_flow]);
309 }
310
311 ChokeFilter choke(&uplink_, all_flow_ids);
312 choke.set_capacity_kbps(capacity_kbps);
313 choke.set_max_delay_ms(max_delay_ms);
314 LinkShare link_share(&choke);
315
316 int64_t one_way_delay_ms = rtt_ms / 2;
317 DelayFilter delay_uplink(&uplink_, all_flow_ids);
318 delay_uplink.SetOneWayDelayMs(one_way_delay_ms);
319
320 JitterFilter jitter(&uplink_, all_flow_ids);
321 jitter.SetMaxJitter(max_jitter_ms);
322
323 std::vector<RateCounterFilter*> rate_counters;
324 for (int flow : media_flow_ids) {
325 rate_counters.push_back(
326 new RateCounterFilter(&uplink_, flow, "Receiver", bwe_names[bwe_type]));
327 }
328 for (int flow : tcp_flow_ids) {
329 rate_counters.push_back(new RateCounterFilter(&uplink_, flow, "Receiver",
330 bwe_names[kTcpEstimator]));
331 }
332
333 RateCounterFilter total_utilization(
334 &uplink_, all_flow_ids, "total_utilization", "Total_link_utilization");
335
336 std::vector<PacketReceiver*> receivers;
337 // Delays is being plotted only for the first flow.
338 // To plot all of them, replace "i == 0" with "true" on new PacketReceiver().
339 for (int media_flow : media_flow_ids) {
340 metric_recorders.push_back(
341 new MetricRecorder(bwe_names[bwe_type], static_cast<int>(media_flow),
342 senders[media_flow], &link_share));
343 receivers.push_back(new PacketReceiver(&uplink_, media_flow, bwe_type,
344 media_flow == 0, false,
345 metric_recorders[media_flow]));
346 metric_recorders[media_flow]->set_plot_available_capacity(
347 media_flow == 0 && plot_total_available_capacity_);
348 metric_recorders[media_flow]->set_start_computing_metrics_ms(max_offset_ms);
349 }
350 // Delays is not being plotted only for TCP flows. To plot all of them,
351 // replace first "false" occurence with "true" on new PacketReceiver().
352 for (int tcp_flow : tcp_flow_ids) {
353 metric_recorders.push_back(
354 new MetricRecorder(bwe_names[kTcpEstimator], static_cast<int>(tcp_flow),
355 senders[tcp_flow], &link_share));
356 receivers.push_back(new PacketReceiver(&uplink_, tcp_flow, kTcpEstimator,
357 false, false,
358 metric_recorders[tcp_flow]));
359 metric_recorders[tcp_flow]->set_plot_available_capacity(
360 tcp_flow == 0 && plot_total_available_capacity_);
361 }
362
363 DelayFilter delay_downlink(&downlink_, all_flow_ids);
364 delay_downlink.SetOneWayDelayMs(one_way_delay_ms);
365
366 RunFor(run_time_seconds * 1000);
367
368 std::map<int, Stats<double>> flow_throughput_kbps;
369 for (RateCounterFilter* rate_counter : rate_counters) {
370 int flow_id = *rate_counter->flow_ids().begin();
371 flow_throughput_kbps[flow_id] = rate_counter->GetBitrateStats();
372 }
373
374 std::map<int, Stats<double>> flow_delay_ms;
375 for (PacketReceiver* receiver : receivers) {
376 int flow_id = *receiver->flow_ids().begin();
377 flow_delay_ms[flow_id] = receiver->GetDelayStats();
378 }
379
380 PrintResults(capacity_kbps, total_utilization.GetBitrateStats(),
381 flow_delay_ms, flow_throughput_kbps);
382
383 if (!field_trial::IsEnabled("WebRTC-QuickPerfTest")) {
384 for (int i : all_flow_ids) {
385 metric_recorders[i]->PlotThroughputHistogram(
386 title, flow_name, static_cast<int>(num_media_flows), 0);
387
388 metric_recorders[i]->PlotLossHistogram(title, flow_name,
389 static_cast<int>(num_media_flows),
390 receivers[i]->GlobalPacketLoss());
391 }
392
393 // Pointless to show delay histogram for TCP flow.
394 for (int i : media_flow_ids) {
395 metric_recorders[i]->PlotDelayHistogram(title, bwe_names[bwe_type],
396 static_cast<int>(num_media_flows),
397 one_way_delay_ms);
398 BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], one_way_delay_ms, i);
399 }
400 }
401
402 for (VideoSource* source : sources)
403 delete source;
404 for (PacketSender* sender : senders)
405 delete sender;
406 for (RateCounterFilter* rate_counter : rate_counters)
407 delete rate_counter;
408 for (PacketReceiver* receiver : receivers)
409 delete receiver;
410 for (MetricRecorder* recorder : metric_recorders)
411 delete recorder;
412 }
413
RunChoke(BandwidthEstimatorType bwe_type,std::vector<int> capacities_kbps)414 void BweTest::RunChoke(BandwidthEstimatorType bwe_type,
415 std::vector<int> capacities_kbps) {
416 int flow_id = bwe_type;
417 AdaptiveVideoSource source(flow_id, 30, 300, 0, 0);
418 VideoSender sender(&uplink_, &source, bwe_type);
419 ChokeFilter choke(&uplink_, flow_id);
420 LinkShare link_share(&choke);
421 MetricRecorder metric_recorder(bwe_names[bwe_type], flow_id, &sender,
422 &link_share);
423 PacketReceiver receiver(&uplink_, flow_id, bwe_type, true, false,
424 &metric_recorder);
425 metric_recorder.set_plot_available_capacity(plot_total_available_capacity_);
426
427 choke.set_max_delay_ms(500);
428 const int64_t kRunTimeMs = 60 * 1000;
429
430 std::stringstream title("Choke");
431 char delimiter = '_';
432
433 for (auto it = capacities_kbps.begin(); it != capacities_kbps.end(); ++it) {
434 choke.set_capacity_kbps(*it);
435 RunFor(kRunTimeMs);
436 title << delimiter << (*it);
437 delimiter = '-';
438 }
439
440 title << "_kbps,_" << (kRunTimeMs / 1000) << "s_each";
441 metric_recorder.PlotThroughputHistogram(title.str(), bwe_names[bwe_type], 1,
442 0);
443 metric_recorder.PlotDelayHistogram(title.str(), bwe_names[bwe_type], 1, 0);
444 // receiver.PlotLossHistogram(title, bwe_names[bwe_type], 1);
445 // receiver.PlotObjectiveHistogram(title, bwe_names[bwe_type], 1);
446 }
447
448 // 5.1. Single Video and Audio media traffic, forward direction.
RunVariableCapacity1SingleFlow(BandwidthEstimatorType bwe_type)449 void BweTest::RunVariableCapacity1SingleFlow(BandwidthEstimatorType bwe_type) {
450 const int kFlowId = 0; // Arbitrary value.
451 AdaptiveVideoSource source(kFlowId, 30, 300, 0, 0);
452 PacedVideoSender sender(&uplink_, &source, bwe_type);
453
454 DefaultEvaluationFilter up_filter(&uplink_, kFlowId);
455 LinkShare link_share(&(up_filter.choke));
456 MetricRecorder metric_recorder(bwe_names[bwe_type], kFlowId, &sender,
457 &link_share);
458
459 PacketReceiver receiver(&uplink_, kFlowId, bwe_type, true, true,
460 &metric_recorder);
461
462 metric_recorder.set_plot_available_capacity(plot_total_available_capacity_);
463
464 DelayFilter down_filter(&downlink_, kFlowId);
465 down_filter.SetOneWayDelayMs(kOneWayDelayMs);
466
467 // Test also with one way propagation delay = 100ms.
468 // up_filter.delay.SetOneWayDelayMs(100);
469 // down_filter.SetOneWayDelayMs(100);
470
471 up_filter.choke.set_capacity_kbps(1000);
472 RunFor(40 * 1000); // 0-40s.
473 up_filter.choke.set_capacity_kbps(2500);
474 RunFor(20 * 1000); // 40-60s.
475 up_filter.choke.set_capacity_kbps(600);
476 RunFor(20 * 1000); // 60-80s.
477 up_filter.choke.set_capacity_kbps(1000);
478 RunFor(20 * 1000); // 80-100s.
479
480 std::string title("5.1_Variable_capacity_single_flow");
481 metric_recorder.PlotThroughputHistogram(title, bwe_names[bwe_type], 1, 0);
482 metric_recorder.PlotDelayHistogram(title, bwe_names[bwe_type], 1,
483 kOneWayDelayMs);
484 metric_recorder.PlotLossHistogram(title, bwe_names[bwe_type], 1,
485 receiver.GlobalPacketLoss());
486 BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, kFlowId);
487 }
488
489 // 5.2. Two forward direction competing flows, variable capacity.
RunVariableCapacity2MultipleFlows(BandwidthEstimatorType bwe_type,size_t num_flows)490 void BweTest::RunVariableCapacity2MultipleFlows(BandwidthEstimatorType bwe_type,
491 size_t num_flows) {
492 std::vector<VideoSource*> sources;
493 std::vector<PacketSender*> senders;
494 std::vector<MetricRecorder*> metric_recorders;
495 std::vector<PacketReceiver*> receivers;
496
497 const int64_t kStartingApartMs = 0; // Flows initialized simultaneously.
498
499 for (size_t i = 0; i < num_flows; ++i) {
500 sources.push_back(new AdaptiveVideoSource(static_cast<int>(i), 30, 300, 0,
501 i * kStartingApartMs));
502 senders.push_back(new VideoSender(&uplink_, sources[i], bwe_type));
503 }
504
505 FlowIds flow_ids = CreateFlowIdRange(0, static_cast<int>(num_flows - 1));
506
507 DefaultEvaluationFilter up_filter(&uplink_, flow_ids);
508 LinkShare link_share(&(up_filter.choke));
509
510 RateCounterFilter total_utilization(&uplink_, flow_ids, "Total_utilization",
511 "Total_link_utilization");
512
513 // Delays is being plotted only for the first flow.
514 // To plot all of them, replace "i == 0" with "true" on new PacketReceiver().
515 for (size_t i = 0; i < num_flows; ++i) {
516 metric_recorders.push_back(new MetricRecorder(
517 bwe_names[bwe_type], static_cast<int>(i), senders[i], &link_share));
518
519 receivers.push_back(new PacketReceiver(&uplink_, static_cast<int>(i),
520 bwe_type, i == 0, false,
521 metric_recorders[i]));
522 metric_recorders[i]->set_plot_available_capacity(
523 i == 0 && plot_total_available_capacity_);
524 }
525
526 DelayFilter down_filter(&downlink_, flow_ids);
527 down_filter.SetOneWayDelayMs(kOneWayDelayMs);
528 // Test also with one way propagation delay = 100ms.
529 // up_filter.delay.SetOneWayDelayMs(100);
530 // down_filter.SetOneWayDelayMs(100);
531
532 up_filter.choke.set_capacity_kbps(4000);
533 RunFor(25 * 1000); // 0-25s.
534 up_filter.choke.set_capacity_kbps(2000);
535 RunFor(25 * 1000); // 25-50s.
536 up_filter.choke.set_capacity_kbps(3500);
537 RunFor(25 * 1000); // 50-75s.
538 up_filter.choke.set_capacity_kbps(1000);
539 RunFor(25 * 1000); // 75-100s.
540 up_filter.choke.set_capacity_kbps(2000);
541 RunFor(25 * 1000); // 100-125s.
542
543 std::string title("5.2_Variable_capacity_two_flows");
544 for (size_t i = 0; i < num_flows; ++i) {
545 metric_recorders[i]->PlotThroughputHistogram(title, bwe_names[bwe_type],
546 num_flows, 0);
547 metric_recorders[i]->PlotDelayHistogram(title, bwe_names[bwe_type],
548 num_flows, kOneWayDelayMs);
549 metric_recorders[i]->PlotLossHistogram(title, bwe_names[bwe_type],
550 num_flows,
551 receivers[i]->GlobalPacketLoss());
552 BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, i);
553 }
554
555 for (VideoSource* source : sources)
556 delete source;
557 for (PacketSender* sender : senders)
558 delete sender;
559 for (MetricRecorder* recorder : metric_recorders)
560 delete recorder;
561 for (PacketReceiver* receiver : receivers)
562 delete receiver;
563 }
564
565 // 5.3. Bi-directional RMCAT flows.
RunBidirectionalFlow(BandwidthEstimatorType bwe_type)566 void BweTest::RunBidirectionalFlow(BandwidthEstimatorType bwe_type) {
567 enum direction { kForward = 0, kBackward };
568 const size_t kNumFlows = 2;
569 std::unique_ptr<AdaptiveVideoSource> sources[kNumFlows];
570 std::unique_ptr<VideoSender> senders[kNumFlows];
571 std::unique_ptr<MetricRecorder> metric_recorders[kNumFlows];
572 std::unique_ptr<PacketReceiver> receivers[kNumFlows];
573
574 sources[kForward].reset(new AdaptiveVideoSource(kForward, 30, 300, 0, 0));
575 senders[kForward].reset(
576 new VideoSender(&uplink_, sources[kForward].get(), bwe_type));
577
578 sources[kBackward].reset(new AdaptiveVideoSource(kBackward, 30, 300, 0, 0));
579 senders[kBackward].reset(
580 new VideoSender(&downlink_, sources[kBackward].get(), bwe_type));
581
582 DefaultEvaluationFilter up_filter(&uplink_, kForward);
583 LinkShare up_link_share(&(up_filter.choke));
584
585 metric_recorders[kForward].reset(new MetricRecorder(
586 bwe_names[bwe_type], kForward, senders[kForward].get(), &up_link_share));
587 receivers[kForward].reset(
588 new PacketReceiver(&uplink_, kForward, bwe_type, true, false,
589 metric_recorders[kForward].get()));
590
591 metric_recorders[kForward].get()->set_plot_available_capacity(
592 plot_total_available_capacity_);
593
594 DefaultEvaluationFilter down_filter(&downlink_, kBackward);
595 LinkShare down_link_share(&(down_filter.choke));
596
597 metric_recorders[kBackward].reset(
598 new MetricRecorder(bwe_names[bwe_type], kBackward,
599 senders[kBackward].get(), &down_link_share));
600 receivers[kBackward].reset(
601 new PacketReceiver(&downlink_, kBackward, bwe_type, true, false,
602 metric_recorders[kBackward].get()));
603
604 metric_recorders[kBackward].get()->set_plot_available_capacity(
605 plot_total_available_capacity_);
606
607 // Test also with one way propagation delay = 100ms.
608 // up_filter.delay.SetOneWayDelayMs(100);
609 // down_filter.delay.SetOneWayDelayMs(100);
610
611 up_filter.choke.set_capacity_kbps(2000);
612 down_filter.choke.set_capacity_kbps(2000);
613 RunFor(20 * 1000); // 0-20s.
614
615 up_filter.choke.set_capacity_kbps(1000);
616 RunFor(15 * 1000); // 20-35s.
617
618 down_filter.choke.set_capacity_kbps(800);
619 RunFor(5 * 1000); // 35-40s.
620
621 up_filter.choke.set_capacity_kbps(500);
622 RunFor(20 * 1000); // 40-60s.
623
624 up_filter.choke.set_capacity_kbps(2000);
625 RunFor(10 * 1000); // 60-70s.
626
627 down_filter.choke.set_capacity_kbps(2000);
628 RunFor(30 * 1000); // 70-100s.
629
630 std::string title("5.3_Bidirectional_flows");
631 for (size_t i = 0; i < kNumFlows; ++i) {
632 metric_recorders[i].get()->PlotThroughputHistogram(
633 title, bwe_names[bwe_type], kNumFlows, 0);
634 metric_recorders[i].get()->PlotDelayHistogram(title, bwe_names[bwe_type],
635 kNumFlows, kOneWayDelayMs);
636 metric_recorders[i].get()->PlotLossHistogram(
637 title, bwe_names[bwe_type], kNumFlows,
638 receivers[i].get()->GlobalPacketLoss());
639 BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, i);
640 }
641 }
642
643 // 5.4. Three forward direction competing flows, constant capacity.
RunSelfFairness(BandwidthEstimatorType bwe_type)644 void BweTest::RunSelfFairness(BandwidthEstimatorType bwe_type) {
645 const int kNumRmcatFlows = 3;
646 const int kNumTcpFlows = 0;
647 const int64_t kRunTimeS = 120;
648 const int kLinkCapacity = 3500;
649
650 int64_t max_delay_ms = kMaxQueueingDelayMs;
651 int64_t rtt_ms = 2 * kOneWayDelayMs;
652
653 const int64_t kStartingApartMs = 20 * 1000;
654 int64_t offsets_ms[kNumRmcatFlows];
655 for (int i = 0; i < kNumRmcatFlows; ++i) {
656 offsets_ms[i] = kStartingApartMs * i;
657 }
658
659 // Test also with one way propagation delay = 100ms.
660 // rtt_ms = 2 * 100;
661 // Test also with bottleneck queue size = 20ms and 1000ms.
662 // max_delay_ms = 20;
663 // max_delay_ms = 1000;
664
665 std::string title("5.4_Self_fairness_test");
666
667 // Test also with one way propagation delay = 100ms.
668 RunFairnessTest(bwe_type, kNumRmcatFlows, kNumTcpFlows, kRunTimeS,
669 kLinkCapacity, max_delay_ms, rtt_ms, kMaxJitterMs, offsets_ms,
670 title, bwe_names[bwe_type]);
671 }
672
673 // 5.5. Five competing RMCAT flows under different RTTs.
RunRoundTripTimeFairness(BandwidthEstimatorType bwe_type)674 void BweTest::RunRoundTripTimeFairness(BandwidthEstimatorType bwe_type) {
675 const int kAllFlowIds[] = {0, 1, 2, 3, 4}; // Five RMCAT flows.
676 const int64_t kAllOneWayDelayMs[] = {10, 25, 50, 100, 150};
677 const size_t kNumFlows = arraysize(kAllFlowIds);
678 std::unique_ptr<AdaptiveVideoSource> sources[kNumFlows];
679 std::unique_ptr<VideoSender> senders[kNumFlows];
680 std::unique_ptr<MetricRecorder> metric_recorders[kNumFlows];
681
682 // Flows initialized 10 seconds apart.
683 const int64_t kStartingApartMs = 10 * 1000;
684
685 for (size_t i = 0; i < kNumFlows; ++i) {
686 sources[i].reset(new AdaptiveVideoSource(kAllFlowIds[i], 30, 300, 0,
687 i * kStartingApartMs));
688 senders[i].reset(new VideoSender(&uplink_, sources[i].get(), bwe_type));
689 }
690
691 ChokeFilter choke_filter(&uplink_, CreateFlowIds(kAllFlowIds, kNumFlows));
692 LinkShare link_share(&choke_filter);
693
694 JitterFilter jitter_filter(&uplink_, CreateFlowIds(kAllFlowIds, kNumFlows));
695
696 std::unique_ptr<DelayFilter> up_delay_filters[kNumFlows];
697 for (size_t i = 0; i < kNumFlows; ++i) {
698 up_delay_filters[i].reset(new DelayFilter(&uplink_, kAllFlowIds[i]));
699 }
700
701 RateCounterFilter total_utilization(
702 &uplink_, CreateFlowIds(kAllFlowIds, kNumFlows), "Total_utilization",
703 "Total_link_utilization");
704
705 // Delays is being plotted only for the first flow.
706 // To plot all of them, replace "i == 0" with "true" on new PacketReceiver().
707 std::unique_ptr<PacketReceiver> receivers[kNumFlows];
708 for (size_t i = 0; i < kNumFlows; ++i) {
709 metric_recorders[i].reset(
710 new MetricRecorder(bwe_names[bwe_type], static_cast<int>(i),
711 senders[i].get(), &link_share));
712
713 receivers[i].reset(new PacketReceiver(&uplink_, kAllFlowIds[i], bwe_type,
714 i == 0, false,
715 metric_recorders[i].get()));
716 metric_recorders[i].get()->set_start_computing_metrics_ms(kStartingApartMs *
717 (kNumFlows - 1));
718 metric_recorders[i].get()->set_plot_available_capacity(
719 i == 0 && plot_total_available_capacity_);
720 }
721
722 std::unique_ptr<DelayFilter> down_delay_filters[kNumFlows];
723 for (size_t i = 0; i < kNumFlows; ++i) {
724 down_delay_filters[i].reset(new DelayFilter(&downlink_, kAllFlowIds[i]));
725 }
726
727 jitter_filter.SetMaxJitter(kMaxJitterMs);
728 choke_filter.set_max_delay_ms(kMaxQueueingDelayMs);
729
730 for (size_t i = 0; i < kNumFlows; ++i) {
731 up_delay_filters[i]->SetOneWayDelayMs(kAllOneWayDelayMs[i]);
732 down_delay_filters[i]->SetOneWayDelayMs(kAllOneWayDelayMs[i]);
733 }
734
735 choke_filter.set_capacity_kbps(3500);
736
737 RunFor(300 * 1000); // 0-300s.
738
739 std::string title("5.5_Round_Trip_Time_Fairness");
740 for (size_t i = 0; i < kNumFlows; ++i) {
741 metric_recorders[i].get()->PlotThroughputHistogram(
742 title, bwe_names[bwe_type], kNumFlows, 0);
743 metric_recorders[i].get()->PlotDelayHistogram(title, bwe_names[bwe_type],
744 kNumFlows, kOneWayDelayMs);
745 metric_recorders[i].get()->PlotLossHistogram(
746 title, bwe_names[bwe_type], kNumFlows,
747 receivers[i].get()->GlobalPacketLoss());
748 BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kAllOneWayDelayMs[i],
749 i);
750 }
751 }
752
753 // 5.6. RMCAT Flow competing with a long TCP Flow.
RunLongTcpFairness(BandwidthEstimatorType bwe_type)754 void BweTest::RunLongTcpFairness(BandwidthEstimatorType bwe_type) {
755 const size_t kNumRmcatFlows = 1;
756 const size_t kNumTcpFlows = 1;
757 const int64_t kRunTimeS = 120;
758 const int kCapacityKbps = 2000;
759 // Tcp starts at t = 0, media flow at t = 5s.
760 const int64_t kOffSetsMs[] = {5000, 0};
761
762 int64_t max_delay_ms = kMaxQueueingDelayMs;
763 int64_t rtt_ms = 2 * kOneWayDelayMs;
764
765 // Test also with one way propagation delay = 100ms.
766 // rtt_ms = 2 * 100;
767 // Test also with bottleneck queue size = 20ms and 1000ms.
768 // max_delay_ms = 20;
769 // max_delay_ms = 1000;
770
771 std::string title("5.6_Long_TCP_Fairness");
772 std::string flow_name = std::string() +
773 bwe_names[bwe_type] + 'x' + bwe_names[kTcpEstimator];
774
775 RunFairnessTest(bwe_type, kNumRmcatFlows, kNumTcpFlows, kRunTimeS,
776 kCapacityKbps, max_delay_ms, rtt_ms, kMaxJitterMs, kOffSetsMs,
777 title, flow_name);
778 }
779
780 // 5.7. RMCAT Flows competing with multiple short TCP Flows.
RunMultipleShortTcpFairness(BandwidthEstimatorType bwe_type,std::vector<int> tcp_file_sizes_bytes,std::vector<int64_t> tcp_starting_times_ms)781 void BweTest::RunMultipleShortTcpFairness(
782 BandwidthEstimatorType bwe_type,
783 std::vector<int> tcp_file_sizes_bytes,
784 std::vector<int64_t> tcp_starting_times_ms) {
785 // Two RMCAT flows and ten TCP flows.
786 const int kAllRmcatFlowIds[] = {0, 1};
787 const int kAllTcpFlowIds[] = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
788
789 assert(tcp_starting_times_ms.size() == tcp_file_sizes_bytes.size() &&
790 tcp_starting_times_ms.size() == arraysize(kAllTcpFlowIds));
791
792 const size_t kNumRmcatFlows = arraysize(kAllRmcatFlowIds);
793 const size_t kNumTotalFlows = kNumRmcatFlows + arraysize(kAllTcpFlowIds);
794
795 std::unique_ptr<AdaptiveVideoSource> sources[kNumRmcatFlows];
796 std::unique_ptr<PacketSender> senders[kNumTotalFlows];
797 std::unique_ptr<MetricRecorder> metric_recorders[kNumTotalFlows];
798 std::unique_ptr<PacketReceiver> receivers[kNumTotalFlows];
799
800 // RMCAT Flows are initialized simultaneosly at t=5 seconds.
801 const int64_t kRmcatStartingTimeMs = 5 * 1000;
802 for (size_t id : kAllRmcatFlowIds) {
803 sources[id].reset(new AdaptiveVideoSource(static_cast<int>(id), 30, 300, 0,
804 kRmcatStartingTimeMs));
805 senders[id].reset(new VideoSender(&uplink_, sources[id].get(), bwe_type));
806 }
807
808 for (size_t id : kAllTcpFlowIds) {
809 senders[id].reset(new TcpSender(&uplink_, static_cast<int>(id),
810 tcp_starting_times_ms[id - kNumRmcatFlows],
811 tcp_file_sizes_bytes[id - kNumRmcatFlows]));
812 }
813
814 FlowIds flow_ids = CreateFlowIdRange(0, static_cast<int>(kNumTotalFlows - 1));
815 DefaultEvaluationFilter up_filter(&uplink_, flow_ids);
816
817 LinkShare link_share(&(up_filter.choke));
818
819 RateCounterFilter total_utilization(&uplink_, flow_ids, "Total_utilization",
820 "Total_link_utilization");
821
822 // Delays is being plotted only for the first flow.
823 // To plot all of them, replace "i == 0" with "true" on new PacketReceiver().
824 for (size_t id : kAllRmcatFlowIds) {
825 metric_recorders[id].reset(
826 new MetricRecorder(bwe_names[bwe_type], static_cast<int>(id),
827 senders[id].get(), &link_share));
828 receivers[id].reset(new PacketReceiver(&uplink_, static_cast<int>(id),
829 bwe_type, id == 0, false,
830 metric_recorders[id].get()));
831 metric_recorders[id].get()->set_start_computing_metrics_ms(
832 kRmcatStartingTimeMs);
833 metric_recorders[id].get()->set_plot_available_capacity(
834 id == 0 && plot_total_available_capacity_);
835 }
836
837 // Delays is not being plotted only for TCP flows. To plot all of them,
838 // replace first "false" occurence with "true" on new PacketReceiver().
839 for (size_t id : kAllTcpFlowIds) {
840 metric_recorders[id].reset(
841 new MetricRecorder(bwe_names[kTcpEstimator], static_cast<int>(id),
842 senders[id].get(), &link_share));
843 receivers[id].reset(new PacketReceiver(&uplink_, static_cast<int>(id),
844 kTcpEstimator, false, false,
845 metric_recorders[id].get()));
846 metric_recorders[id].get()->set_plot_available_capacity(
847 id == 0 && plot_total_available_capacity_);
848 }
849
850 DelayFilter down_filter(&downlink_, flow_ids);
851 down_filter.SetOneWayDelayMs(kOneWayDelayMs);
852
853 // Test also with one way propagation delay = 100ms.
854 // up_filter.delay.SetOneWayDelayMs(100);
855 // down_filter.SetOneWayDelayms(100);
856
857 // Test also with bottleneck queue size = 20ms and 1000ms.
858 // up_filter.choke.set_max_delay_ms(20);
859 // up_filter.choke.set_max_delay_ms(1000);
860
861 // Test also with no Jitter:
862 // up_filter.jitter.SetMaxJitter(0);
863
864 up_filter.choke.set_capacity_kbps(2000);
865
866 RunFor(300 * 1000); // 0-300s.
867
868 std::string title("5.7_Multiple_short_TCP_flows");
869 for (size_t id : kAllRmcatFlowIds) {
870 metric_recorders[id].get()->PlotThroughputHistogram(
871 title, bwe_names[bwe_type], kNumRmcatFlows, 0);
872 metric_recorders[id].get()->PlotDelayHistogram(
873 title, bwe_names[bwe_type], kNumRmcatFlows, kOneWayDelayMs);
874 metric_recorders[id].get()->PlotLossHistogram(
875 title, bwe_names[bwe_type], kNumRmcatFlows,
876 receivers[id].get()->GlobalPacketLoss());
877 BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, id);
878 }
879 }
880
881 // 5.8. Three forward direction competing flows, constant capacity.
882 // During the test, one of the flows is paused and later resumed.
RunPauseResumeFlows(BandwidthEstimatorType bwe_type)883 void BweTest::RunPauseResumeFlows(BandwidthEstimatorType bwe_type) {
884 const int kAllFlowIds[] = {0, 1, 2}; // Three RMCAT flows.
885 const size_t kNumFlows = arraysize(kAllFlowIds);
886
887 std::unique_ptr<AdaptiveVideoSource> sources[kNumFlows];
888 std::unique_ptr<VideoSender> senders[kNumFlows];
889 std::unique_ptr<MetricRecorder> metric_recorders[kNumFlows];
890 std::unique_ptr<PacketReceiver> receivers[kNumFlows];
891
892 // Flows initialized simultaneously.
893 const int64_t kStartingApartMs = 0;
894
895 for (size_t i = 0; i < kNumFlows; ++i) {
896 sources[i].reset(new AdaptiveVideoSource(kAllFlowIds[i], 30, 300, 0,
897 i * kStartingApartMs));
898 senders[i].reset(new VideoSender(&uplink_, sources[i].get(), bwe_type));
899 }
900
901 DefaultEvaluationFilter filter(&uplink_,
902 CreateFlowIds(kAllFlowIds, kNumFlows));
903
904 LinkShare link_share(&(filter.choke));
905
906 RateCounterFilter total_utilization(
907 &uplink_, CreateFlowIds(kAllFlowIds, kNumFlows), "Total_utilization",
908 "Total_link_utilization");
909
910 // Delays is being plotted only for the first flow.
911 // To plot all of them, replace "i == 0" with "true" on new PacketReceiver().
912 for (size_t i = 0; i < kNumFlows; ++i) {
913 metric_recorders[i].reset(
914 new MetricRecorder(bwe_names[bwe_type], static_cast<int>(i),
915 senders[i].get(), &link_share));
916 receivers[i].reset(new PacketReceiver(&uplink_, kAllFlowIds[i], bwe_type,
917 i == 0, false,
918 metric_recorders[i].get()));
919 metric_recorders[i].get()->set_start_computing_metrics_ms(kStartingApartMs *
920 (kNumFlows - 1));
921 metric_recorders[i].get()->set_plot_available_capacity(
922 i == 0 && plot_total_available_capacity_);
923 }
924
925 // Test also with one way propagation delay = 100ms.
926 // filter.delay.SetOneWayDelayMs(100);
927 filter.choke.set_capacity_kbps(3500);
928
929 RunFor(40 * 1000); // 0-40s.
930 senders[0].get()->Pause();
931 RunFor(20 * 1000); // 40-60s.
932 senders[0].get()->Resume(20 * 1000);
933 RunFor(60 * 1000); // 60-120s.
934
935 int64_t paused[] = {20 * 1000, 0, 0};
936
937 // First flow is being paused, hence having a different optimum.
938 const std::string optima_lines[] = {"1", "2", "2"};
939
940 std::string title("5.8_Pause_and_resume_media_flow");
941 for (size_t i = 0; i < kNumFlows; ++i) {
942 metric_recorders[i].get()->PlotThroughputHistogram(
943 title, bwe_names[bwe_type], kNumFlows, paused[i], optima_lines[i]);
944 metric_recorders[i].get()->PlotDelayHistogram(title, bwe_names[bwe_type],
945 kNumFlows, kOneWayDelayMs);
946 metric_recorders[i].get()->PlotLossHistogram(
947 title, bwe_names[bwe_type], kNumFlows,
948 receivers[i].get()->GlobalPacketLoss());
949 BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, i);
950 }
951 }
952
953 // Following functions are used for randomizing TCP file size and
954 // starting time, used on 5.7 RunMultipleShortTcpFairness.
955 // They are pseudo-random generators, creating always the same
956 // value sequence for a given Random seed.
957
GetFileSizesBytes(int num_files)958 std::vector<int> BweTest::GetFileSizesBytes(int num_files) {
959 // File size chosen from uniform distribution between [100,1000] kB.
960 const int kMinKbytes = 100;
961 const int kMaxKbytes = 1000;
962
963 Random random(0x12345678);
964 std::vector<int> tcp_file_sizes_bytes;
965
966 while (num_files-- > 0) {
967 tcp_file_sizes_bytes.push_back(random.Rand(kMinKbytes, kMaxKbytes) * 1000);
968 }
969
970 return tcp_file_sizes_bytes;
971 }
972
GetStartingTimesMs(int num_files)973 std::vector<int64_t> BweTest::GetStartingTimesMs(int num_files) {
974 // OFF state behaves as an exp. distribution with mean = 10 seconds.
975 const float kMeanMs = 10000.0f;
976 Random random(0x12345678);
977
978 std::vector<int64_t> tcp_starting_times_ms;
979
980 // Two TCP Flows are initialized simultaneosly at t=0 seconds.
981 for (int i = 0; i < 2; ++i, --num_files) {
982 tcp_starting_times_ms.push_back(0);
983 }
984
985 // Other TCP Flows are initialized in an OFF state.
986 while (num_files-- > 0) {
987 tcp_starting_times_ms.push_back(
988 static_cast<int64_t>(random.Exponential(1.0f / kMeanMs)));
989 }
990
991 return tcp_starting_times_ms;
992 }
993
994 } // namespace bwe
995 } // namespace testing
996 } // namespace webrtc
997