1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "components/download/internal/common/parallel_download_utils.h"
6 
7 #include <map>
8 #include <memory>
9 
10 #include "base/strings/string_number_conversions.h"
11 #include "base/test/scoped_feature_list.h"
12 #include "components/download/public/common/download_features.h"
13 #include "components/download/public/common/download_file_impl.h"
14 #include "components/download/public/common/download_save_info.h"
15 #include "components/download/public/common/mock_input_stream.h"
16 #include "components/download/public/common/parallel_download_configs.h"
17 #include "testing/gmock/include/gmock/gmock.h"
18 #include "testing/gtest/include/gtest/gtest.h"
19 
20 using ::testing::Return;
21 using ::testing::StrictMock;
22 
23 namespace download {
24 
25 namespace {
26 
27 const int kErrorStreamOffset = 100;
28 
29 }  // namespace
30 
31 class ParallelDownloadUtilsTest : public testing::Test {};
32 
33 class ParallelDownloadUtilsRecoverErrorTest
34     : public ::testing::TestWithParam<int64_t> {
35  public:
ParallelDownloadUtilsRecoverErrorTest()36   ParallelDownloadUtilsRecoverErrorTest() : input_stream_(nullptr) {}
37 
38   // Creates a source stream to test.
CreateSourceStream(int64_t offset)39   std::unique_ptr<DownloadFileImpl::SourceStream> CreateSourceStream(
40       int64_t offset) {
41     input_stream_ = new StrictMock<MockInputStream>();
42     EXPECT_CALL(*input_stream_, GetCompletionStatus())
43         .WillRepeatedly(Return(DOWNLOAD_INTERRUPT_REASON_NONE));
44     return std::make_unique<DownloadFileImpl::SourceStream>(
45         offset, offset, std::unique_ptr<MockInputStream>(input_stream_));
46   }
47 
48  protected:
49   // Stream for sending data into the SourceStream.
50   StrictMock<MockInputStream>* input_stream_;
51 };
52 
TEST_F(ParallelDownloadUtilsTest,FindSlicesToDownload)53 TEST_F(ParallelDownloadUtilsTest, FindSlicesToDownload) {
54   std::vector<DownloadItem::ReceivedSlice> downloaded_slices;
55   std::vector<DownloadItem::ReceivedSlice> slices_to_download =
56       FindSlicesToDownload(downloaded_slices);
57   EXPECT_EQ(1u, slices_to_download.size());
58   EXPECT_EQ(0, slices_to_download[0].offset);
59   EXPECT_EQ(DownloadSaveInfo::kLengthFullContent,
60             slices_to_download[0].received_bytes);
61 
62   downloaded_slices.emplace_back(0, 500);
63   slices_to_download = FindSlicesToDownload(downloaded_slices);
64   EXPECT_EQ(1u, slices_to_download.size());
65   EXPECT_EQ(500, slices_to_download[0].offset);
66   EXPECT_EQ(DownloadSaveInfo::kLengthFullContent,
67             slices_to_download[0].received_bytes);
68 
69   // Create a gap between slices.
70   downloaded_slices.emplace_back(1000, 500);
71   slices_to_download = FindSlicesToDownload(downloaded_slices);
72   EXPECT_EQ(2u, slices_to_download.size());
73   EXPECT_EQ(500, slices_to_download[0].offset);
74   EXPECT_EQ(500, slices_to_download[0].received_bytes);
75   EXPECT_EQ(1500, slices_to_download[1].offset);
76   EXPECT_EQ(DownloadSaveInfo::kLengthFullContent,
77             slices_to_download[1].received_bytes);
78 
79   // Fill the gap.
80   downloaded_slices.emplace(downloaded_slices.begin() + 1,
81                             slices_to_download[0]);
82   slices_to_download = FindSlicesToDownload(downloaded_slices);
83   EXPECT_EQ(1u, slices_to_download.size());
84   EXPECT_EQ(1500, slices_to_download[0].offset);
85   EXPECT_EQ(DownloadSaveInfo::kLengthFullContent,
86             slices_to_download[0].received_bytes);
87 
88   // Create a new gap at the beginning.
89   downloaded_slices.erase(downloaded_slices.begin());
90   slices_to_download = FindSlicesToDownload(downloaded_slices);
91   EXPECT_EQ(2u, slices_to_download.size());
92   EXPECT_EQ(0, slices_to_download[0].offset);
93   EXPECT_EQ(500, slices_to_download[0].received_bytes);
94   EXPECT_EQ(1500, slices_to_download[1].offset);
95   EXPECT_EQ(DownloadSaveInfo::kLengthFullContent,
96             slices_to_download[1].received_bytes);
97 }
98 
TEST_F(ParallelDownloadUtilsTest,AddOrMergeReceivedSliceIntoSortedArray)99 TEST_F(ParallelDownloadUtilsTest, AddOrMergeReceivedSliceIntoSortedArray) {
100   std::vector<DownloadItem::ReceivedSlice> slices;
101   DownloadItem::ReceivedSlice slice1(500, 500);
102   EXPECT_EQ(0u, AddOrMergeReceivedSliceIntoSortedArray(slice1, slices));
103   EXPECT_EQ(1u, slices.size());
104   EXPECT_EQ(slice1, slices[0]);
105 
106   // Adding a slice that can be merged with existing slice.
107   DownloadItem::ReceivedSlice slice2(1000, 400);
108   EXPECT_EQ(0u, AddOrMergeReceivedSliceIntoSortedArray(slice2, slices));
109   EXPECT_EQ(1u, slices.size());
110   EXPECT_EQ(500, slices[0].offset);
111   EXPECT_EQ(900, slices[0].received_bytes);
112 
113   DownloadItem::ReceivedSlice slice3(0, 50);
114   EXPECT_EQ(0u, AddOrMergeReceivedSliceIntoSortedArray(slice3, slices));
115   EXPECT_EQ(2u, slices.size());
116   EXPECT_EQ(slice3, slices[0]);
117 
118   DownloadItem::ReceivedSlice slice4(100, 50);
119   EXPECT_EQ(1u, AddOrMergeReceivedSliceIntoSortedArray(slice4, slices));
120   EXPECT_EQ(3u, slices.size());
121   EXPECT_EQ(slice3, slices[0]);
122   EXPECT_EQ(slice4, slices[1]);
123 
124   // A new slice can only merge with an existing slice earlier in the file, not
125   // later in the file.
126   DownloadItem::ReceivedSlice slice5(50, 50);
127   EXPECT_EQ(0u, AddOrMergeReceivedSliceIntoSortedArray(slice5, slices));
128   EXPECT_EQ(3u, slices.size());
129   EXPECT_EQ(0, slices[0].offset);
130   EXPECT_EQ(100, slices[0].received_bytes);
131   EXPECT_EQ(slice4, slices[1]);
132 }
133 
134 // Verify if a preceding stream can recover the download for half open error
135 // stream(the current last stream).
TEST_P(ParallelDownloadUtilsRecoverErrorTest,RecoverErrorForHalfOpenErrorStream)136 TEST_P(ParallelDownloadUtilsRecoverErrorTest,
137        RecoverErrorForHalfOpenErrorStream) {
138   // Create a stream that will work on byte range "100-".
139   const int kErrorStreamOffset = 100;
140 
141   auto error_stream = CreateSourceStream(kErrorStreamOffset);
142   error_stream->set_finished(true);
143 
144   // Get starting offset of preceding stream.
145   int64_t preceding_offset = GetParam();
146   EXPECT_LT(preceding_offset, kErrorStreamOffset);
147   auto preceding_stream = CreateSourceStream(preceding_offset);
148   // Half open preceding stream can always recover the error for later streams.
149   EXPECT_FALSE(preceding_stream->is_finished());
150   EXPECT_EQ(0u, preceding_stream->bytes_written());
151   EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
152 
153   // Half open finished preceding stream with 0 bytes written, if there is no
154   // error, the download should be finished.
155   preceding_stream->set_finished(true);
156   EXPECT_EQ(DOWNLOAD_INTERRUPT_REASON_NONE,
157             preceding_stream->GetCompletionStatus());
158   EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
159 
160   // Half open finished preceding stream with error, should be treated as
161   // failed.
162   EXPECT_CALL(*input_stream_, GetCompletionStatus())
163       .WillRepeatedly(Return(DOWNLOAD_INTERRUPT_REASON_FILE_NO_SPACE));
164   EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
165 
166   // Even if it has written some data.
167   preceding_stream->OnBytesConsumed(1000u, 1000u);
168   EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
169 
170   int64_t bytes_consumed = kErrorStreamOffset - preceding_offset - 1;
171   // Half open successfully finished preceding stream should always be
172   // able to recover error, even if it is not reaching the error offset as the
173   // error stream might be requesting something our of range.
174   preceding_stream = CreateSourceStream(preceding_offset);
175   preceding_stream->set_finished(false);
176   EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
177   preceding_stream->set_finished(true);
178   preceding_stream->OnBytesConsumed(bytes_consumed, bytes_consumed);
179   EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
180   preceding_stream->OnBytesConsumed(1, 1);
181   EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
182 
183   // If the preceding stream is truncated, it should never be able to recover
184   // a half open stream.
185   preceding_stream = CreateSourceStream(preceding_offset);
186   preceding_stream->TruncateLengthWithWrittenDataBlock(kErrorStreamOffset, 1);
187   EXPECT_EQ(preceding_stream->length(), kErrorStreamOffset - preceding_offset);
188   EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
189   preceding_stream->set_finished(true);
190   EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
191   preceding_stream->OnBytesConsumed(bytes_consumed, bytes_consumed);
192   EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
193   preceding_stream->OnBytesConsumed(1, 1);
194   EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
195 }
196 
197 // Verify recovery for length capped error stream.
TEST_P(ParallelDownloadUtilsRecoverErrorTest,RecoverErrorForLengthCappedErrorStream)198 TEST_P(ParallelDownloadUtilsRecoverErrorTest,
199        RecoverErrorForLengthCappedErrorStream) {
200   // Create a stream that will work on byte range "100-150".
201   const int kErrorStreamLength = 50;
202   auto error_stream = CreateSourceStream(kErrorStreamOffset);
203   error_stream->TruncateLengthWithWrittenDataBlock(
204       kErrorStreamOffset + kErrorStreamLength, 1);
205   EXPECT_EQ(error_stream->length(), 50);
206   error_stream->set_finished(true);
207 
208   // Get starting offset of preceding stream.
209   const int64_t preceding_offset = GetParam();
210   EXPECT_LT(preceding_offset, kErrorStreamOffset);
211 
212   // Create an half open preceding stream.
213   auto preceding_stream = CreateSourceStream(preceding_offset);
214   EXPECT_FALSE(preceding_stream->is_finished());
215   EXPECT_EQ(0u, preceding_stream->bytes_written());
216 
217   // Since the preceding stream can reach the starting offset, it should be able
218   // to recover the error stream..
219   EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
220   int64_t bytes_consumed = kErrorStreamOffset - preceding_offset;
221   preceding_stream->OnBytesConsumed(bytes_consumed, bytes_consumed);
222   EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
223   preceding_stream->OnBytesConsumed(kErrorStreamLength - 1,
224                                     kErrorStreamLength - 1);
225   EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
226   preceding_stream->OnBytesConsumed(1, 1);
227   EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
228 
229   // If preceding stream is truncated after error stream, checks data written.
230   preceding_stream = CreateSourceStream(preceding_offset);
231   preceding_stream->TruncateLengthWithWrittenDataBlock(
232       kErrorStreamOffset + kErrorStreamLength, 1);
233   EXPECT_EQ(preceding_stream->length(),
234             kErrorStreamOffset + kErrorStreamLength - preceding_offset);
235   EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
236   preceding_stream->set_finished(true);
237   EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
238   preceding_stream->OnBytesConsumed(bytes_consumed, bytes_consumed);
239   EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
240   preceding_stream->OnBytesConsumed(kErrorStreamLength - 1,
241                                     kErrorStreamLength - 1);
242   EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
243   preceding_stream->OnBytesConsumed(1, 1);
244   EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
245 
246   // Even if inject an error, since data written has cover the upper bound of
247   // the error stream, it should succeed.
248   EXPECT_CALL(*input_stream_, GetCompletionStatus())
249       .WillRepeatedly(Return(DOWNLOAD_INTERRUPT_REASON_FILE_NO_SPACE));
250   EXPECT_TRUE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
251 
252   // If preceding stream is truncated before or in the middle of error stream,
253   // it should not recover the error stream when it reaches its length.
254   preceding_stream = CreateSourceStream(preceding_offset);
255   preceding_stream->TruncateLengthWithWrittenDataBlock(kErrorStreamOffset + 1,
256                                                        1);
257   EXPECT_EQ(preceding_stream->length(),
258             kErrorStreamOffset + 1 - preceding_offset);
259   EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
260   preceding_stream->set_finished(true);
261   EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
262   preceding_stream->OnBytesConsumed(bytes_consumed + 1, bytes_consumed + 1);
263   EXPECT_FALSE(CanRecoverFromError(error_stream.get(), preceding_stream.get()));
264 }
265 
266 // The testing value specified offset for preceding stream. The error stream
267 // offset is fixed value.
268 INSTANTIATE_TEST_SUITE_P(ParallelDownloadUtilsTestSuite,
269                          ParallelDownloadUtilsRecoverErrorTest,
270                          ::testing::Values(0, 20, 80));
271 
272 // Ensure the minimum slice size is correctly applied.
TEST_F(ParallelDownloadUtilsTest,FindSlicesForRemainingContentMinSliceSize)273 TEST_F(ParallelDownloadUtilsTest, FindSlicesForRemainingContentMinSliceSize) {
274   // Minimum slice size is smaller than total length, only one slice returned.
275   DownloadItem::ReceivedSlices slices =
276       FindSlicesForRemainingContent(0, 100, 3, 150);
277   EXPECT_EQ(1u, slices.size());
278   EXPECT_EQ(0, slices[0].offset);
279   EXPECT_EQ(0, slices[0].received_bytes);
280 
281   // Request count is large, the minimum slice size should limit the number of
282   // slices returned.
283   slices = FindSlicesForRemainingContent(0, 100, 33, 50);
284   EXPECT_EQ(2u, slices.size());
285   EXPECT_EQ(0, slices[0].offset);
286   EXPECT_EQ(50, slices[0].received_bytes);
287   EXPECT_EQ(50, slices[1].offset);
288   EXPECT_EQ(0, slices[1].received_bytes);
289 
290   // Can chunk 2 slices under minimum slice size, but request count is only 1,
291   // request count should win.
292   slices = FindSlicesForRemainingContent(0, 100, 1, 50);
293   EXPECT_EQ(1u, slices.size());
294   EXPECT_EQ(0, slices[0].offset);
295   EXPECT_EQ(0, slices[0].received_bytes);
296 
297   // A total 100 bytes data and a 51 bytes minimum slice size, only one slice is
298   // returned.
299   slices = FindSlicesForRemainingContent(0, 100, 3, 51);
300   EXPECT_EQ(1u, slices.size());
301   EXPECT_EQ(0, slices[0].offset);
302   EXPECT_EQ(0, slices[0].received_bytes);
303 
304   // Extreme case where size is smaller than request number.
305   slices = FindSlicesForRemainingContent(0, 1, 3, 1);
306   EXPECT_EQ(1u, slices.size());
307   EXPECT_EQ(DownloadItem::ReceivedSlice(0, 0), slices[0]);
308 
309   // Normal case.
310   slices = FindSlicesForRemainingContent(0, 100, 3, 5);
311   EXPECT_EQ(3u, slices.size());
312   EXPECT_EQ(DownloadItem::ReceivedSlice(0, 33), slices[0]);
313   EXPECT_EQ(DownloadItem::ReceivedSlice(33, 33), slices[1]);
314   EXPECT_EQ(DownloadItem::ReceivedSlice(66, 0), slices[2]);
315 }
316 
TEST_F(ParallelDownloadUtilsTest,GetMaxContiguousDataBlockSizeFromBeginning)317 TEST_F(ParallelDownloadUtilsTest, GetMaxContiguousDataBlockSizeFromBeginning) {
318   std::vector<DownloadItem::ReceivedSlice> slices;
319   slices.emplace_back(500, 500);
320   EXPECT_EQ(0, GetMaxContiguousDataBlockSizeFromBeginning(slices));
321 
322   DownloadItem::ReceivedSlice slice1(0, 200);
323   AddOrMergeReceivedSliceIntoSortedArray(slice1, slices);
324   EXPECT_EQ(200, GetMaxContiguousDataBlockSizeFromBeginning(slices));
325 
326   DownloadItem::ReceivedSlice slice2(200, 300);
327   AddOrMergeReceivedSliceIntoSortedArray(slice2, slices);
328   EXPECT_EQ(1000, GetMaxContiguousDataBlockSizeFromBeginning(slices));
329 }
330 
331 // Test to verify Finch parameters for enabled experiment group is read
332 // correctly.
TEST_F(ParallelDownloadUtilsTest,FinchConfigEnabled)333 TEST_F(ParallelDownloadUtilsTest, FinchConfigEnabled) {
334   base::test::ScopedFeatureList feature_list;
335   std::map<std::string, std::string> params = {
336       {kMinSliceSizeFinchKey, "1234"},
337       {kParallelRequestCountFinchKey, "6"},
338       {kParallelRequestDelayFinchKey, "2000"},
339       {kParallelRequestRemainingTimeFinchKey, "3"}};
340   feature_list.InitAndEnableFeatureWithParameters(
341       features::kParallelDownloading, params);
342   EXPECT_TRUE(IsParallelDownloadEnabled());
343   EXPECT_EQ(GetMinSliceSizeConfig(), 1234);
344   EXPECT_EQ(GetParallelRequestCountConfig(), 6);
345   EXPECT_EQ(GetParallelRequestDelayConfig(), base::TimeDelta::FromSeconds(2));
346   EXPECT_EQ(GetParallelRequestRemainingTimeConfig(),
347             base::TimeDelta::FromSeconds(3));
348 }
349 
350 // Test to verify the disable experiment group will actually disable the
351 // feature.
TEST_F(ParallelDownloadUtilsTest,FinchConfigDisabled)352 TEST_F(ParallelDownloadUtilsTest, FinchConfigDisabled) {
353   base::test::ScopedFeatureList feature_list;
354   feature_list.InitAndDisableFeature(features::kParallelDownloading);
355   EXPECT_FALSE(IsParallelDownloadEnabled());
356 }
357 
358 // Test to verify that the Finch parameter |enable_parallel_download| works
359 // correctly.
TEST_F(ParallelDownloadUtilsTest,FinchConfigDisabledWithParameter)360 TEST_F(ParallelDownloadUtilsTest, FinchConfigDisabledWithParameter) {
361   {
362     base::test::ScopedFeatureList feature_list;
363     std::map<std::string, std::string> params = {
364         {kMinSliceSizeFinchKey, "4321"},
365         {kEnableParallelDownloadFinchKey, "false"}};
366     feature_list.InitAndEnableFeatureWithParameters(
367         features::kParallelDownloading, params);
368     // Use |enable_parallel_download| to disable parallel download in enabled
369     // experiment group.
370     EXPECT_FALSE(IsParallelDownloadEnabled());
371     EXPECT_EQ(GetMinSliceSizeConfig(), 4321);
372   }
373   {
374     base::test::ScopedFeatureList feature_list;
375     std::map<std::string, std::string> params = {
376         {kMinSliceSizeFinchKey, "4321"},
377         {kEnableParallelDownloadFinchKey, "true"}};
378     feature_list.InitAndEnableFeatureWithParameters(
379         features::kParallelDownloading, params);
380     // Disable only if |enable_parallel_download| sets to false.
381     EXPECT_TRUE(IsParallelDownloadEnabled());
382     EXPECT_EQ(GetMinSliceSizeConfig(), 4321);
383   }
384   {
385     base::test::ScopedFeatureList feature_list;
386     std::map<std::string, std::string> params = {
387         {kMinSliceSizeFinchKey, "4321"}};
388     feature_list.InitAndEnableFeatureWithParameters(
389         features::kParallelDownloading, params);
390     // Empty |enable_parallel_download| in an enabled experiment group will have
391     // no impact.
392     EXPECT_TRUE(IsParallelDownloadEnabled());
393     EXPECT_EQ(GetMinSliceSizeConfig(), 4321);
394   }
395 }
396 
397 }  // namespace download
398