1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "components/download/internal/common/parallel_download_utils.h"
6
7 #include <map>
8 #include <memory>
9
10 #include "base/strings/string_number_conversions.h"
11 #include "base/test/scoped_feature_list.h"
12 #include "components/download/public/common/download_features.h"
13 #include "components/download/public/common/download_file_impl.h"
14 #include "components/download/public/common/download_save_info.h"
15 #include "components/download/public/common/mock_input_stream.h"
16 #include "components/download/public/common/parallel_download_configs.h"
17 #include "testing/gmock/include/gmock/gmock.h"
18 #include "testing/gtest/include/gtest/gtest.h"
19
20 using ::testing::Return;
21 using ::testing::StrictMock;
22
23 namespace download {
24
25 namespace {
26
27 const int kErrorStreamOffset = 100;
28
29 } // namespace
30
31 class ParallelDownloadUtilsTest : public testing::Test {};
32
33 class ParallelDownloadUtilsRecoverErrorTest
34 : public ::testing::TestWithParam<int64_t> {
35 public:
ParallelDownloadUtilsRecoverErrorTest()36 ParallelDownloadUtilsRecoverErrorTest() : input_stream_(nullptr) {}
37
38 // Creates a source stream to test.
CreateSourceStream(int64_t offset)39 std::unique_ptr<DownloadFileImpl::SourceStream> CreateSourceStream(
40 int64_t offset) {
41 input_stream_ = new StrictMock<MockInputStream>();
42 EXPECT_CALL(*input_stream_, GetCompletionStatus())
43 .WillRepeatedly(Return(DOWNLOAD_INTERRUPT_REASON_NONE));
44 return std::make_unique<DownloadFileImpl::SourceStream>(
45 offset, offset, std::unique_ptr<MockInputStream>(input_stream_));
46 }
47
48 protected:
49 // Stream for sending data into the SourceStream.
50 StrictMock<MockInputStream>* input_stream_;
51 };
52
TEST_F(ParallelDownloadUtilsTest,FindSlicesToDownload)53 TEST_F(ParallelDownloadUtilsTest, FindSlicesToDownload) {
54 std::vector<DownloadItem::ReceivedSlice> downloaded_slices;
55 std::vector<DownloadItem::ReceivedSlice> slices_to_download =
56 FindSlicesToDownload(downloaded_slices);
57 EXPECT_EQ(1u, slices_to_download.size());
58 EXPECT_EQ(0, slices_to_download[0].offset);
59 EXPECT_EQ(DownloadSaveInfo::kLengthFullContent,
60 slices_to_download[0].received_bytes);
61
62 downloaded_slices.emplace_back(0, 500);
63 slices_to_download = FindSlicesToDownload(downloaded_slices);
64 EXPECT_EQ(1u, slices_to_download.size());
65 EXPECT_EQ(500, slices_to_download[0].offset);
66 EXPECT_EQ(DownloadSaveInfo::kLengthFullContent,
67 slices_to_download[0].received_bytes);
68
69 // Create a gap between slices.
70 downloaded_slices.emplace_back(1000, 500);
71 slices_to_download = FindSlicesToDownload(downloaded_slices);
72 EXPECT_EQ(2u, slices_to_download.size());
73 EXPECT_EQ(500, slices_to_download[0].offset);
74 EXPECT_EQ(500, slices_to_download[0].received_bytes);
75 EXPECT_EQ(1500, slices_to_download[1].offset);
76 EXPECT_EQ(DownloadSaveInfo::kLengthFullContent,
77 slices_to_download[1].received_bytes);
78
79 // Fill the gap.
80 downloaded_slices.emplace(downloaded_slices.begin() + 1,
81 slices_to_download[0]);
82 slices_to_download = FindSlicesToDownload(downloaded_slices);
83 EXPECT_EQ(1u, slices_to_download.size());
84 EXPECT_EQ(1500, slices_to_download[0].offset);
85 EXPECT_EQ(DownloadSaveInfo::kLengthFullContent,
86 slices_to_download[0].received_bytes);
87
88 // Create a new gap at the beginning.
89 downloaded_slices.erase(downloaded_slices.begin());
90 slices_to_download = FindSlicesToDownload(downloaded_slices);
91 EXPECT_EQ(2u, slices_to_download.size());
92 EXPECT_EQ(0, slices_to_download[0].offset);
93 EXPECT_EQ(500, slices_to_download[0].received_bytes);
94 EXPECT_EQ(1500, slices_to_download[1].offset);
95 EXPECT_EQ(DownloadSaveInfo::kLengthFullContent,
96 slices_to_download[1].received_bytes);
97 }
98
TEST_F(ParallelDownloadUtilsTest,AddOrMergeReceivedSliceIntoSortedArray)99 TEST_F(ParallelDownloadUtilsTest, AddOrMergeReceivedSliceIntoSortedArray) {
100 std::vector<DownloadItem::ReceivedSlice> slices;
101 DownloadItem::ReceivedSlice slice1(500, 500);
102 EXPECT_EQ(0u, AddOrMergeReceivedSliceIntoSortedArray(slice1, slices));
103 EXPECT_EQ(1u, slices.size());
104 EXPECT_EQ(slice1, slices[0]);
105
106 // Adding a slice that can be merged with existing slice.
107 DownloadItem::ReceivedSlice slice2(1000, 400);
108 EXPECT_EQ(0u, AddOrMergeReceivedSliceIntoSortedArray(slice2, slices));
109 EXPECT_EQ(1u, slices.size());
110 EXPECT_EQ(500, slices[0].offset);
111 EXPECT_EQ(900, slices[0].received_bytes);
112
113 DownloadItem::ReceivedSlice slice3(0, 50);
114 EXPECT_EQ(0u, AddOrMergeReceivedSliceIntoSortedArray(slice3, slices));
115 EXPECT_EQ(2u, slices.size());
116 EXPECT_EQ(slice3, slices[0]);
117
118 DownloadItem::ReceivedSlice slice4(100, 50);
119 EXPECT_EQ(1u, AddOrMergeReceivedSliceIntoSortedArray(slice4, slices));
120 EXPECT_EQ(3u, slices.size());
121 EXPECT_EQ(slice3, slices[0]);
122 EXPECT_EQ(slice4, slices[1]);
123
124 // A new slice can only merge with an existing slice earlier in the file, not
125 // later in the file.
126 DownloadItem::ReceivedSlice slice5(50, 50);
127 EXPECT_EQ(0u, AddOrMergeReceivedSliceIntoSortedArray(slice5, slices));
128 EXPECT_EQ(3u, slices.size());
129 EXPECT_EQ(0, slices[0].offset);
130 EXPECT_EQ(100, slices[0].received_bytes);
131 EXPECT_EQ(slice4, slices[1]);
132 }
133
134 // Verify if a preceding stream can recover the download for half open error
135 // stream(the current last stream).
TEST_P(ParallelDownloadUtilsRecoverErrorTest,RecoverErrorForHalfOpenErrorStream)136 TEST_P(ParallelDownloadUtilsRecoverErrorTest,
137 RecoverErrorForHalfOpenErrorStream) {
138 // Create a stream that will work on byte range "100-".
139 const int kErrorStreamOffset = 100;
140
141 auto error_stream = CreateSourceStream(kErrorStreamOffset);
142 error_stream->set_finished(true);
143
144 // Get starting offset of preceding stream.
145 int64_t preceding_offset = GetParam();
146 EXPECT_LT(preceding_offset, kErrorStreamOffset);
147 auto preceding_stream = CreateSourceStream(preceding_offset);
148 // Half open preceding stream can always recover the error for later streams.
149 EXPECT_FALSE(preceding_stream->is_finished());
150 EXPECT_EQ(0u, preceding_stream->bytes_written());
151 EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
152
153 // Half open finished preceding stream with 0 bytes written, if there is no
154 // error, the download should be finished.
155 preceding_stream->set_finished(true);
156 EXPECT_EQ(DOWNLOAD_INTERRUPT_REASON_NONE,
157 preceding_stream->GetCompletionStatus());
158 EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
159
160 // Half open finished preceding stream with error, should be treated as
161 // failed.
162 EXPECT_CALL(*input_stream_, GetCompletionStatus())
163 .WillRepeatedly(Return(DOWNLOAD_INTERRUPT_REASON_FILE_NO_SPACE));
164 EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
165
166 // Even if it has written some data.
167 preceding_stream->OnBytesConsumed(1000u, 1000u);
168 EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
169
170 int64_t bytes_consumed = kErrorStreamOffset - preceding_offset - 1;
171 // Half open successfully finished preceding stream should always be
172 // able to recover error, even if it is not reaching the error offset as the
173 // error stream might be requesting something our of range.
174 preceding_stream = CreateSourceStream(preceding_offset);
175 preceding_stream->set_finished(false);
176 EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
177 preceding_stream->set_finished(true);
178 preceding_stream->OnBytesConsumed(bytes_consumed, bytes_consumed);
179 EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
180 preceding_stream->OnBytesConsumed(1, 1);
181 EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
182
183 // If the preceding stream is truncated, it should never be able to recover
184 // a half open stream.
185 preceding_stream = CreateSourceStream(preceding_offset);
186 preceding_stream->TruncateLengthWithWrittenDataBlock(kErrorStreamOffset, 1);
187 EXPECT_EQ(preceding_stream->length(), kErrorStreamOffset - preceding_offset);
188 EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
189 preceding_stream->set_finished(true);
190 EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
191 preceding_stream->OnBytesConsumed(bytes_consumed, bytes_consumed);
192 EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
193 preceding_stream->OnBytesConsumed(1, 1);
194 EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
195 }
196
197 // Verify recovery for length capped error stream.
TEST_P(ParallelDownloadUtilsRecoverErrorTest,RecoverErrorForLengthCappedErrorStream)198 TEST_P(ParallelDownloadUtilsRecoverErrorTest,
199 RecoverErrorForLengthCappedErrorStream) {
200 // Create a stream that will work on byte range "100-150".
201 const int kErrorStreamLength = 50;
202 auto error_stream = CreateSourceStream(kErrorStreamOffset);
203 error_stream->TruncateLengthWithWrittenDataBlock(
204 kErrorStreamOffset + kErrorStreamLength, 1);
205 EXPECT_EQ(error_stream->length(), 50);
206 error_stream->set_finished(true);
207
208 // Get starting offset of preceding stream.
209 const int64_t preceding_offset = GetParam();
210 EXPECT_LT(preceding_offset, kErrorStreamOffset);
211
212 // Create an half open preceding stream.
213 auto preceding_stream = CreateSourceStream(preceding_offset);
214 EXPECT_FALSE(preceding_stream->is_finished());
215 EXPECT_EQ(0u, preceding_stream->bytes_written());
216
217 // Since the preceding stream can reach the starting offset, it should be able
218 // to recover the error stream..
219 EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
220 int64_t bytes_consumed = kErrorStreamOffset - preceding_offset;
221 preceding_stream->OnBytesConsumed(bytes_consumed, bytes_consumed);
222 EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
223 preceding_stream->OnBytesConsumed(kErrorStreamLength - 1,
224 kErrorStreamLength - 1);
225 EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
226 preceding_stream->OnBytesConsumed(1, 1);
227 EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
228
229 // If preceding stream is truncated after error stream, checks data written.
230 preceding_stream = CreateSourceStream(preceding_offset);
231 preceding_stream->TruncateLengthWithWrittenDataBlock(
232 kErrorStreamOffset + kErrorStreamLength, 1);
233 EXPECT_EQ(preceding_stream->length(),
234 kErrorStreamOffset + kErrorStreamLength - preceding_offset);
235 EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
236 preceding_stream->set_finished(true);
237 EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
238 preceding_stream->OnBytesConsumed(bytes_consumed, bytes_consumed);
239 EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
240 preceding_stream->OnBytesConsumed(kErrorStreamLength - 1,
241 kErrorStreamLength - 1);
242 EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
243 preceding_stream->OnBytesConsumed(1, 1);
244 EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
245
246 // Even if inject an error, since data written has cover the upper bound of
247 // the error stream, it should succeed.
248 EXPECT_CALL(*input_stream_, GetCompletionStatus())
249 .WillRepeatedly(Return(DOWNLOAD_INTERRUPT_REASON_FILE_NO_SPACE));
250 EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
251
252 // If preceding stream is truncated before or in the middle of error stream,
253 // it should not recover the error stream when it reaches its length.
254 preceding_stream = CreateSourceStream(preceding_offset);
255 preceding_stream->TruncateLengthWithWrittenDataBlock(kErrorStreamOffset + 1,
256 1);
257 EXPECT_EQ(preceding_stream->length(),
258 kErrorStreamOffset + 1 - preceding_offset);
259 EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
260 preceding_stream->set_finished(true);
261 EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
262 preceding_stream->OnBytesConsumed(bytes_consumed + 1, bytes_consumed + 1);
263 EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
264 }
265
266 // The testing value specified offset for preceding stream. The error stream
267 // offset is fixed value.
268 INSTANTIATE_TEST_SUITE_P(ParallelDownloadUtilsTestSuite,
269 ParallelDownloadUtilsRecoverErrorTest,
270 ::testing::Values(0, 20, 80));
271
272 // Ensure the minimum slice size is correctly applied.
TEST_F(ParallelDownloadUtilsTest,FindSlicesForRemainingContentMinSliceSize)273 TEST_F(ParallelDownloadUtilsTest, FindSlicesForRemainingContentMinSliceSize) {
274 // Minimum slice size is smaller than total length, only one slice returned.
275 DownloadItem::ReceivedSlices slices =
276 FindSlicesForRemainingContent(0, 100, 3, 150);
277 EXPECT_EQ(1u, slices.size());
278 EXPECT_EQ(0, slices[0].offset);
279 EXPECT_EQ(0, slices[0].received_bytes);
280
281 // Request count is large, the minimum slice size should limit the number of
282 // slices returned.
283 slices = FindSlicesForRemainingContent(0, 100, 33, 50);
284 EXPECT_EQ(2u, slices.size());
285 EXPECT_EQ(0, slices[0].offset);
286 EXPECT_EQ(50, slices[0].received_bytes);
287 EXPECT_EQ(50, slices[1].offset);
288 EXPECT_EQ(0, slices[1].received_bytes);
289
290 // Can chunk 2 slices under minimum slice size, but request count is only 1,
291 // request count should win.
292 slices = FindSlicesForRemainingContent(0, 100, 1, 50);
293 EXPECT_EQ(1u, slices.size());
294 EXPECT_EQ(0, slices[0].offset);
295 EXPECT_EQ(0, slices[0].received_bytes);
296
297 // A total 100 bytes data and a 51 bytes minimum slice size, only one slice is
298 // returned.
299 slices = FindSlicesForRemainingContent(0, 100, 3, 51);
300 EXPECT_EQ(1u, slices.size());
301 EXPECT_EQ(0, slices[0].offset);
302 EXPECT_EQ(0, slices[0].received_bytes);
303
304 // Extreme case where size is smaller than request number.
305 slices = FindSlicesForRemainingContent(0, 1, 3, 1);
306 EXPECT_EQ(1u, slices.size());
307 EXPECT_EQ(DownloadItem::ReceivedSlice(0, 0), slices[0]);
308
309 // Normal case.
310 slices = FindSlicesForRemainingContent(0, 100, 3, 5);
311 EXPECT_EQ(3u, slices.size());
312 EXPECT_EQ(DownloadItem::ReceivedSlice(0, 33), slices[0]);
313 EXPECT_EQ(DownloadItem::ReceivedSlice(33, 33), slices[1]);
314 EXPECT_EQ(DownloadItem::ReceivedSlice(66, 0), slices[2]);
315 }
316
TEST_F(ParallelDownloadUtilsTest,GetMaxContiguousDataBlockSizeFromBeginning)317 TEST_F(ParallelDownloadUtilsTest, GetMaxContiguousDataBlockSizeFromBeginning) {
318 std::vector<DownloadItem::ReceivedSlice> slices;
319 slices.emplace_back(500, 500);
320 EXPECT_EQ(0, GetMaxContiguousDataBlockSizeFromBeginning(slices));
321
322 DownloadItem::ReceivedSlice slice1(0, 200);
323 AddOrMergeReceivedSliceIntoSortedArray(slice1, slices);
324 EXPECT_EQ(200, GetMaxContiguousDataBlockSizeFromBeginning(slices));
325
326 DownloadItem::ReceivedSlice slice2(200, 300);
327 AddOrMergeReceivedSliceIntoSortedArray(slice2, slices);
328 EXPECT_EQ(1000, GetMaxContiguousDataBlockSizeFromBeginning(slices));
329 }
330
331 // Test to verify Finch parameters for enabled experiment group is read
332 // correctly.
TEST_F(ParallelDownloadUtilsTest,FinchConfigEnabled)333 TEST_F(ParallelDownloadUtilsTest, FinchConfigEnabled) {
334 base::test::ScopedFeatureList feature_list;
335 std::map<std::string, std::string> params = {
336 {kMinSliceSizeFinchKey, "1234"},
337 {kParallelRequestCountFinchKey, "6"},
338 {kParallelRequestDelayFinchKey, "2000"},
339 {kParallelRequestRemainingTimeFinchKey, "3"}};
340 feature_list.InitAndEnableFeatureWithParameters(
341 features::kParallelDownloading, params);
342 EXPECT_TRUE(IsParallelDownloadEnabled());
343 EXPECT_EQ(GetMinSliceSizeConfig(), 1234);
344 EXPECT_EQ(GetParallelRequestCountConfig(), 6);
345 EXPECT_EQ(GetParallelRequestDelayConfig(), base::TimeDelta::FromSeconds(2));
346 EXPECT_EQ(GetParallelRequestRemainingTimeConfig(),
347 base::TimeDelta::FromSeconds(3));
348 }
349
350 // Test to verify the disable experiment group will actually disable the
351 // feature.
TEST_F(ParallelDownloadUtilsTest,FinchConfigDisabled)352 TEST_F(ParallelDownloadUtilsTest, FinchConfigDisabled) {
353 base::test::ScopedFeatureList feature_list;
354 feature_list.InitAndDisableFeature(features::kParallelDownloading);
355 EXPECT_FALSE(IsParallelDownloadEnabled());
356 }
357
358 // Test to verify that the Finch parameter |enable_parallel_download| works
359 // correctly.
TEST_F(ParallelDownloadUtilsTest,FinchConfigDisabledWithParameter)360 TEST_F(ParallelDownloadUtilsTest, FinchConfigDisabledWithParameter) {
361 {
362 base::test::ScopedFeatureList feature_list;
363 std::map<std::string, std::string> params = {
364 {kMinSliceSizeFinchKey, "4321"},
365 {kEnableParallelDownloadFinchKey, "false"}};
366 feature_list.InitAndEnableFeatureWithParameters(
367 features::kParallelDownloading, params);
368 // Use |enable_parallel_download| to disable parallel download in enabled
369 // experiment group.
370 EXPECT_FALSE(IsParallelDownloadEnabled());
371 EXPECT_EQ(GetMinSliceSizeConfig(), 4321);
372 }
373 {
374 base::test::ScopedFeatureList feature_list;
375 std::map<std::string, std::string> params = {
376 {kMinSliceSizeFinchKey, "4321"},
377 {kEnableParallelDownloadFinchKey, "true"}};
378 feature_list.InitAndEnableFeatureWithParameters(
379 features::kParallelDownloading, params);
380 // Disable only if |enable_parallel_download| sets to false.
381 EXPECT_TRUE(IsParallelDownloadEnabled());
382 EXPECT_EQ(GetMinSliceSizeConfig(), 4321);
383 }
384 {
385 base::test::ScopedFeatureList feature_list;
386 std::map<std::string, std::string> params = {
387 {kMinSliceSizeFinchKey, "4321"}};
388 feature_list.InitAndEnableFeatureWithParameters(
389 features::kParallelDownloading, params);
390 // Empty |enable_parallel_download| in an enabled experiment group will have
391 // no impact.
392 EXPECT_TRUE(IsParallelDownloadEnabled());
393 EXPECT_EQ(GetMinSliceSizeConfig(), 4321);
394 }
395 }
396
397 } // namespace download
398