1 /* <!-- copyright */
2 /*
3 * aria2 - The high speed download utility
4 *
5 * Copyright (C) 2006 Tatsuhiro Tsujikawa
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 *
21 * In addition, as a special exception, the copyright holders give
22 * permission to link the code of portions of this program with the
23 * OpenSSL library under certain conditions as described in each
24 * individual source file, and distribute linked combinations
25 * including the two.
26 * You must obey the GNU General Public License in all respects
27 * for all of the code used other than OpenSSL. If you modify
28 * file(s) with this exception, you may extend this exception to your
29 * version of the file(s), but you are not obligated to do so. If you
30 * do not wish to do so, delete this exception statement from your
31 * version. If you delete this exception statement from all source
32 * files in the program, then also delete it here.
33 */
34 /* copyright --> */
35 #include "Piece.h"
36
37 #include <array>
38 #include <cassert>
39
40 #include "util.h"
41 #include "BitfieldMan.h"
42 #include "A2STR.h"
43 #include "util.h"
44 #include "a2functional.h"
45 #include "WrDiskCache.h"
46 #include "WrDiskCacheEntry.h"
47 #include "LogFactory.h"
48 #include "fmt.h"
49 #include "DiskAdaptor.h"
50 #include "MessageDigest.h"
51
52 namespace aria2 {
53
Piece()54 Piece::Piece() : index_(0), length_(0), nextBegin_(0), usedBySegment_(false) {}
55
Piece(size_t index,int64_t length,int32_t blockLength)56 Piece::Piece(size_t index, int64_t length, int32_t blockLength)
57 : bitfield_(make_unique<BitfieldMan>(blockLength, length)),
58 index_(index),
59 length_(length),
60 nextBegin_(0),
61 usedBySegment_(false)
62 {
63 }
64
65 Piece::~Piece() = default;
66
completeBlock(size_t blockIndex)67 void Piece::completeBlock(size_t blockIndex)
68 {
69 bitfield_->setBit(blockIndex);
70 bitfield_->unsetUseBit(blockIndex);
71 }
72
clearAllBlock(WrDiskCache * diskCache)73 void Piece::clearAllBlock(WrDiskCache* diskCache)
74 {
75 bitfield_->clearAllBit();
76 bitfield_->clearAllUseBit();
77 if (diskCache && wrCache_) {
78 clearWrCache(diskCache);
79 }
80 }
81
setAllBlock()82 void Piece::setAllBlock() { bitfield_->setAllBit(); }
83
pieceComplete() const84 bool Piece::pieceComplete() const { return bitfield_->isAllBitSet(); }
85
countBlock() const86 size_t Piece::countBlock() const { return bitfield_->countBlock(); }
87
getBlockLength(size_t index) const88 int32_t Piece::getBlockLength(size_t index) const
89 {
90 return bitfield_->getBlockLength(index);
91 }
92
getBlockLength() const93 int32_t Piece::getBlockLength() const { return bitfield_->getBlockLength(); }
94
getBitfield() const95 const unsigned char* Piece::getBitfield() const
96 {
97 return bitfield_->getBitfield();
98 }
99
getBitfieldLength() const100 size_t Piece::getBitfieldLength() const
101 {
102 return bitfield_->getBitfieldLength();
103 }
104
isBlockUsed(size_t index) const105 bool Piece::isBlockUsed(size_t index) const
106 {
107 return bitfield_->isUseBitSet(index);
108 }
109
cancelBlock(size_t blockIndex)110 void Piece::cancelBlock(size_t blockIndex)
111 {
112 bitfield_->unsetUseBit(blockIndex);
113 }
114
countCompleteBlock() const115 size_t Piece::countCompleteBlock() const
116 {
117 return bitfield_->countBlock() - bitfield_->countMissingBlock();
118 }
119
countMissingBlock() const120 size_t Piece::countMissingBlock() const
121 {
122 return bitfield_->countMissingBlock();
123 }
124
hasBlock(size_t blockIndex) const125 bool Piece::hasBlock(size_t blockIndex) const
126 {
127 return bitfield_->isBitSet(blockIndex);
128 }
129
getMissingUnusedBlockIndex(size_t & index) const130 bool Piece::getMissingUnusedBlockIndex(size_t& index) const
131 {
132 if (bitfield_->getFirstMissingUnusedIndex(index)) {
133 bitfield_->setUseBit(index);
134 return true;
135 }
136 else {
137 return false;
138 }
139 }
140
getMissingUnusedBlockIndex(std::vector<size_t> & indexes,size_t n) const141 size_t Piece::getMissingUnusedBlockIndex(std::vector<size_t>& indexes,
142 size_t n) const
143 {
144 size_t num = bitfield_->getFirstNMissingUnusedIndex(indexes, n);
145 if (num) {
146 for (std::vector<size_t>::const_iterator i = indexes.end() - num,
147 eoi = indexes.end();
148 i != eoi; ++i) {
149 bitfield_->setUseBit(*i);
150 }
151 }
152 return num;
153 }
154
getFirstMissingBlockIndexWithoutLock(size_t & index) const155 bool Piece::getFirstMissingBlockIndexWithoutLock(size_t& index) const
156 {
157 return bitfield_->getFirstMissingIndex(index);
158 }
159
getAllMissingBlockIndexes(unsigned char * misbitfield,size_t mislen) const160 bool Piece::getAllMissingBlockIndexes(unsigned char* misbitfield,
161 size_t mislen) const
162 {
163 return bitfield_->getAllMissingIndexes(misbitfield, mislen);
164 }
165
toString() const166 std::string Piece::toString() const
167 {
168 return fmt("piece: index=%lu, length=%" PRId64,
169 static_cast<unsigned long>(index_), length_);
170 }
171
reconfigure(int64_t length)172 void Piece::reconfigure(int64_t length)
173 {
174 length_ = length;
175 // TODO currently, this function is only called from
176 // GrowSegment::updateWrittenLength(). If we use default block
177 // length (16K), and length_ gets large (e.g., 4GB), creating
178 // BitfieldMan for each call is very expensive. Therefore, we use
179 // maximum block length for now to reduce the overhead. Ideally, we
180 // check the code thoroughly and remove bitfield_ if we can.
181 bitfield_ =
182 make_unique<BitfieldMan>(std::numeric_limits<int32_t>::max(), length_);
183 }
184
setBitfield(const unsigned char * bitfield,size_t len)185 void Piece::setBitfield(const unsigned char* bitfield, size_t len)
186 {
187 bitfield_->setBitfield(bitfield, len);
188 }
189
getCompletedLength()190 int64_t Piece::getCompletedLength() { return bitfield_->getCompletedLength(); }
191
setHashType(const std::string & hashType)192 void Piece::setHashType(const std::string& hashType) { hashType_ = hashType; }
193
updateHash(int64_t begin,const unsigned char * data,size_t dataLength)194 bool Piece::updateHash(int64_t begin, const unsigned char* data,
195 size_t dataLength)
196 {
197 if (hashType_.empty()) {
198 return false;
199 }
200 if (begin == nextBegin_ &&
201 nextBegin_ + static_cast<int64_t>(dataLength) <= length_) {
202 if (!mdctx_) {
203 mdctx_ = MessageDigest::create(hashType_);
204 }
205 mdctx_->update(data, dataLength);
206 nextBegin_ += dataLength;
207 return true;
208 }
209 else {
210 return false;
211 }
212 }
213
isHashCalculated() const214 bool Piece::isHashCalculated() const { return mdctx_ && nextBegin_ == length_; }
215
getDigest()216 std::string Piece::getDigest()
217 {
218 if (!mdctx_) {
219 return A2STR::NIL;
220 }
221 else {
222 std::string hash = mdctx_->digest();
223 destroyHashContext();
224 return hash;
225 }
226 }
227
228 namespace {
updateHashWithRead(MessageDigest * mdctx,const std::shared_ptr<DiskAdaptor> & adaptor,int64_t offset,size_t len)229 void updateHashWithRead(MessageDigest* mdctx,
230 const std::shared_ptr<DiskAdaptor>& adaptor,
231 int64_t offset, size_t len)
232 {
233 std::array<unsigned char, 4_k> buf;
234 ldiv_t res = ldiv(len, buf.size());
235 for (int j = 0; j < res.quot; ++j) {
236 ssize_t nread = adaptor->readData(buf.data(), buf.size(), offset);
237 if ((size_t)nread != buf.size()) {
238 throw DL_ABORT_EX(fmt(EX_FILE_READ, "n/a", "data is too short"));
239 }
240 mdctx->update(buf.data(), nread);
241 offset += nread;
242 }
243 if (res.rem) {
244 ssize_t nread = adaptor->readData(buf.data(), res.rem, offset);
245 if (nread != res.rem) {
246 throw DL_ABORT_EX(fmt(EX_FILE_READ, "n/a", "data is too short"));
247 }
248 mdctx->update(buf.data(), nread);
249 }
250 }
251 } // namespace
252
253 std::string
getDigestWithWrCache(size_t pieceLength,const std::shared_ptr<DiskAdaptor> & adaptor)254 Piece::getDigestWithWrCache(size_t pieceLength,
255 const std::shared_ptr<DiskAdaptor>& adaptor)
256 {
257 auto mdctx = MessageDigest::create(hashType_);
258 int64_t start = static_cast<int64_t>(index_) * pieceLength;
259 int64_t goff = start;
260 if (wrCache_) {
261 const WrDiskCacheEntry::DataCellSet& dataSet = wrCache_->getDataSet();
262 for (auto& d : dataSet) {
263 if (goff < d->goff) {
264 updateHashWithRead(mdctx.get(), adaptor, goff, d->goff - goff);
265 }
266 mdctx->update(d->data + d->offset, d->len);
267 goff = d->goff + d->len;
268 }
269 updateHashWithRead(mdctx.get(), adaptor, goff, start + length_ - goff);
270 }
271 else {
272 updateHashWithRead(mdctx.get(), adaptor, goff, length_);
273 }
274 return mdctx->digest();
275 }
276
destroyHashContext()277 void Piece::destroyHashContext()
278 {
279 mdctx_.reset();
280 nextBegin_ = 0;
281 }
282
usedBy(cuid_t cuid) const283 bool Piece::usedBy(cuid_t cuid) const
284 {
285 return std::find(users_.begin(), users_.end(), cuid) != users_.end();
286 }
287
addUser(cuid_t cuid)288 void Piece::addUser(cuid_t cuid)
289 {
290 if (std::find(users_.begin(), users_.end(), cuid) == users_.end()) {
291 users_.push_back(cuid);
292 }
293 }
294
removeUser(cuid_t cuid)295 void Piece::removeUser(cuid_t cuid)
296 {
297 users_.erase(std::remove(users_.begin(), users_.end(), cuid), users_.end());
298 }
299
initWrCache(WrDiskCache * diskCache,const std::shared_ptr<DiskAdaptor> & diskAdaptor)300 void Piece::initWrCache(WrDiskCache* diskCache,
301 const std::shared_ptr<DiskAdaptor>& diskAdaptor)
302 {
303 if (!diskCache) {
304 return;
305 }
306 assert(!wrCache_);
307 wrCache_ = make_unique<WrDiskCacheEntry>(diskAdaptor);
308 bool rv = diskCache->add(wrCache_.get());
309 assert(rv);
310 }
311
flushWrCache(WrDiskCache * diskCache)312 void Piece::flushWrCache(WrDiskCache* diskCache)
313 {
314 if (!diskCache) {
315 return;
316 }
317 assert(wrCache_);
318 ssize_t size = static_cast<ssize_t>(wrCache_->getSize());
319 diskCache->update(wrCache_.get(), -size);
320 wrCache_->writeToDisk();
321 }
322
clearWrCache(WrDiskCache * diskCache)323 void Piece::clearWrCache(WrDiskCache* diskCache)
324 {
325 if (!diskCache) {
326 return;
327 }
328 assert(wrCache_);
329 ssize_t size = static_cast<ssize_t>(wrCache_->getSize());
330 diskCache->update(wrCache_.get(), -size);
331 wrCache_->clear();
332 }
333
updateWrCache(WrDiskCache * diskCache,unsigned char * data,size_t offset,size_t len,size_t capacity,int64_t goff)334 void Piece::updateWrCache(WrDiskCache* diskCache, unsigned char* data,
335 size_t offset, size_t len, size_t capacity,
336 int64_t goff)
337 {
338 if (!diskCache) {
339 return;
340 }
341 assert(wrCache_);
342 A2_LOG_DEBUG(fmt("updateWrCache entry=%p", wrCache_.get()));
343 auto cell = new WrDiskCacheEntry::DataCell();
344 cell->goff = goff;
345 cell->data = data;
346 cell->offset = offset;
347 cell->len = len;
348 cell->capacity = capacity;
349 bool rv;
350 rv = wrCache_->cacheData(cell);
351 assert(rv);
352 rv = diskCache->update(wrCache_.get(), len);
353 assert(rv);
354 }
355
appendWrCache(WrDiskCache * diskCache,int64_t goff,const unsigned char * data,size_t len)356 size_t Piece::appendWrCache(WrDiskCache* diskCache, int64_t goff,
357 const unsigned char* data, size_t len)
358 {
359 if (!diskCache) {
360 return 0;
361 }
362 assert(wrCache_);
363 size_t delta = wrCache_->append(goff, data, len);
364 bool rv;
365 if (delta > 0) {
366 rv = diskCache->update(wrCache_.get(), delta);
367 assert(rv);
368 }
369 return delta;
370 }
371
releaseWrCache(WrDiskCache * diskCache)372 void Piece::releaseWrCache(WrDiskCache* diskCache)
373 {
374 if (diskCache && wrCache_) {
375 diskCache->remove(wrCache_.get());
376 wrCache_.reset();
377 }
378 }
379
380 } // namespace aria2
381