1 //
2 // Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2021
3 //
4 // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 //
7 #include "td/telegram/files/FileManager.h"
8
9 #include "td/telegram/ConfigShared.h"
10 #include "td/telegram/FileReferenceManager.h"
11 #include "td/telegram/files/FileData.h"
12 #include "td/telegram/files/FileDb.h"
13 #include "td/telegram/files/FileLoaderUtils.h"
14 #include "td/telegram/files/FileLocation.h"
15 #include "td/telegram/files/FileLocation.hpp"
16 #include "td/telegram/Global.h"
17 #include "td/telegram/logevent/LogEvent.h"
18 #include "td/telegram/misc.h"
19 #include "td/telegram/SecureStorage.h"
20 #include "td/telegram/TdDb.h"
21 #include "td/telegram/TdParameters.h"
22 #include "td/telegram/Version.h"
23
24 #include "td/actor/SleepActor.h"
25
26 #include "td/utils/algorithm.h"
27 #include "td/utils/base64.h"
28 #include "td/utils/crypto.h"
29 #include "td/utils/filesystem.h"
30 #include "td/utils/format.h"
31 #include "td/utils/HttpUrl.h"
32 #include "td/utils/logging.h"
33 #include "td/utils/misc.h"
34 #include "td/utils/PathView.h"
35 #include "td/utils/port/FileFd.h"
36 #include "td/utils/port/path.h"
37 #include "td/utils/port/Stat.h"
38 #include "td/utils/ScopeGuard.h"
39 #include "td/utils/SliceBuilder.h"
40 #include "td/utils/StringBuilder.h"
41 #include "td/utils/Time.h"
42 #include "td/utils/tl_helpers.h"
43 #include "td/utils/tl_parsers.h"
44
45 #include <algorithm>
46 #include <cmath>
47 #include <limits>
48 #include <numeric>
49 #include <tuple>
50 #include <unordered_set>
51 #include <utility>
52
53 namespace td {
54 namespace {
55 constexpr int64 MAX_FILE_SIZE = 2000 * (1 << 20) /* 2000MB */;
56 } // namespace
57
58 int VERBOSITY_NAME(update_file) = VERBOSITY_NAME(INFO);
59
operator <<(StringBuilder & string_builder,FileLocationSource source)60 StringBuilder &operator<<(StringBuilder &string_builder, FileLocationSource source) {
61 switch (source) {
62 case FileLocationSource::None:
63 return string_builder << "None";
64 case FileLocationSource::FromUser:
65 return string_builder << "User";
66 case FileLocationSource::FromBinlog:
67 return string_builder << "Binlog";
68 case FileLocationSource::FromDatabase:
69 return string_builder << "Database";
70 case FileLocationSource::FromServer:
71 return string_builder << "Server";
72 default:
73 UNREACHABLE();
74 return string_builder << "Unknown";
75 }
76 }
77
operator <<(StringBuilder & string_builder,FileManager::Query::Type type)78 StringBuilder &operator<<(StringBuilder &string_builder, FileManager::Query::Type type) {
79 switch (type) {
80 case FileManager::Query::Type::UploadByHash:
81 return string_builder << "UploadByHash";
82 case FileManager::Query::Type::UploadWaitFileReference:
83 return string_builder << "UploadWaitFileReference";
84 case FileManager::Query::Type::Upload:
85 return string_builder << "Upload";
86 case FileManager::Query::Type::DownloadWaitFileReference:
87 return string_builder << "DownloadWaitFileReference";
88 case FileManager::Query::Type::DownloadReloadDialog:
89 return string_builder << "DownloadReloadDialog";
90 case FileManager::Query::Type::Download:
91 return string_builder << "Download";
92 case FileManager::Query::Type::SetContent:
93 return string_builder << "SetContent";
94 case FileManager::Query::Type::Generate:
95 return string_builder << "Generate";
96 default:
97 UNREACHABLE();
98 return string_builder << "Unknown";
99 }
100 }
101
NewRemoteFileLocation(RemoteFileLocation remote,FileLocationSource source)102 NewRemoteFileLocation::NewRemoteFileLocation(RemoteFileLocation remote, FileLocationSource source) {
103 switch (remote.type()) {
104 case RemoteFileLocation::Type::Empty:
105 break;
106 case RemoteFileLocation::Type::Partial:
107 partial = make_unique<PartialRemoteFileLocation>(remote.partial());
108 break;
109 case RemoteFileLocation::Type::Full:
110 full = remote.full();
111 full_source = source;
112 is_full_alive = true;
113 break;
114 default:
115 UNREACHABLE();
116 }
117 }
118
partial_or_empty() const119 RemoteFileLocation NewRemoteFileLocation::partial_or_empty() const {
120 if (partial) {
121 return RemoteFileLocation(*partial);
122 }
123 return {};
124 }
125
operator ->() const126 FileNode *FileNodePtr::operator->() const {
127 return get();
128 }
129
operator *() const130 FileNode &FileNodePtr::operator*() const {
131 return *get();
132 }
133
get() const134 FileNode *FileNodePtr::get() const {
135 auto res = get_unsafe();
136 CHECK(res);
137 return res;
138 }
139
get_remote() const140 FullRemoteFileLocation *FileNodePtr::get_remote() const {
141 return file_manager_->get_remote(file_id_.get_remote());
142 }
143
get_unsafe() const144 FileNode *FileNodePtr::get_unsafe() const {
145 CHECK(file_manager_ != nullptr);
146 return file_manager_->get_file_node_raw(file_id_);
147 }
148
operator bool() const149 FileNodePtr::operator bool() const {
150 return file_manager_ != nullptr && get_unsafe() != nullptr;
151 }
152
recalc_ready_prefix_size(int64 prefix_offset,int64 ready_prefix_size)153 void FileNode::recalc_ready_prefix_size(int64 prefix_offset, int64 ready_prefix_size) {
154 if (local_.type() != LocalFileLocation::Type::Partial) {
155 return;
156 }
157 int64 new_local_ready_prefix_size;
158 if (download_offset_ == prefix_offset) {
159 new_local_ready_prefix_size = ready_prefix_size;
160 } else {
161 new_local_ready_prefix_size = Bitmask(Bitmask::Decode{}, local_.partial().ready_bitmask_)
162 .get_ready_prefix_size(download_offset_, local_.partial().part_size_, size_);
163 }
164 if (new_local_ready_prefix_size != local_ready_prefix_size_) {
165 VLOG(update_file) << "File " << main_file_id_ << " has changed local_ready_prefix_size from "
166 << local_ready_prefix_size_ << " to " << new_local_ready_prefix_size;
167 local_ready_prefix_size_ = new_local_ready_prefix_size;
168 on_info_changed();
169 }
170 }
171
init_ready_size()172 void FileNode::init_ready_size() {
173 if (local_.type() != LocalFileLocation::Type::Partial) {
174 return;
175 }
176 auto bitmask = Bitmask(Bitmask::Decode{}, local_.partial().ready_bitmask_);
177 local_ready_prefix_size_ = bitmask.get_ready_prefix_size(0, local_.partial().part_size_, size_);
178 local_ready_size_ = bitmask.get_total_size(local_.partial().part_size_, size_);
179 }
180
set_download_offset(int64 download_offset)181 void FileNode::set_download_offset(int64 download_offset) {
182 if (download_offset < 0 || download_offset > MAX_FILE_SIZE) {
183 return;
184 }
185 if (download_offset == download_offset_) {
186 return;
187 }
188
189 VLOG(update_file) << "File " << main_file_id_ << " has changed download_offset from " << download_offset_ << " to "
190 << download_offset;
191 download_offset_ = download_offset;
192 is_download_offset_dirty_ = true;
193 recalc_ready_prefix_size(-1, -1);
194 on_info_changed();
195 }
set_download_limit(int64 download_limit)196 void FileNode::set_download_limit(int64 download_limit) {
197 if (download_limit < 0) {
198 return;
199 }
200 if (download_limit == download_limit_) {
201 return;
202 }
203
204 VLOG(update_file) << "File " << main_file_id_ << " has changed download_limit from " << download_limit_ << " to "
205 << download_limit;
206 download_limit_ = download_limit;
207 is_download_limit_dirty_ = true;
208 }
209
drop_local_location()210 void FileNode::drop_local_location() {
211 set_local_location(LocalFileLocation(), 0, -1, -1);
212 }
213
set_local_location(const LocalFileLocation & local,int64 ready_size,int64 prefix_offset,int64 ready_prefix_size)214 void FileNode::set_local_location(const LocalFileLocation &local, int64 ready_size, int64 prefix_offset,
215 int64 ready_prefix_size) {
216 if (local_ready_size_ != ready_size) {
217 VLOG(update_file) << "File " << main_file_id_ << " has changed local ready size from " << local_ready_size_
218 << " to " << ready_size;
219 local_ready_size_ = ready_size;
220 on_info_changed();
221 }
222 if (local_ != local) {
223 VLOG(update_file) << "File " << main_file_id_ << " has changed local location";
224 local_ = local;
225
226 recalc_ready_prefix_size(prefix_offset, ready_prefix_size);
227
228 on_changed();
229 }
230 }
231
set_new_remote_location(NewRemoteFileLocation new_remote)232 void FileNode::set_new_remote_location(NewRemoteFileLocation new_remote) {
233 if (new_remote.full) {
234 if (remote_.full && remote_.full.value() == new_remote.full.value()) {
235 if (remote_.full.value().get_access_hash() != new_remote.full.value().get_access_hash() ||
236 remote_.full.value().get_file_reference() != new_remote.full.value().get_file_reference() ||
237 remote_.full.value().get_source() != new_remote.full.value().get_source()) {
238 on_pmc_changed();
239 }
240 } else {
241 VLOG(update_file) << "File " << main_file_id_ << " has changed remote location";
242 on_changed();
243 }
244 remote_.full = new_remote.full;
245 remote_.full_source = new_remote.full_source;
246 remote_.is_full_alive = new_remote.is_full_alive;
247 } else {
248 if (remote_.full) {
249 VLOG(update_file) << "File " << main_file_id_ << " has lost remote location";
250 remote_.full = {};
251 remote_.is_full_alive = false;
252 remote_.full_source = FileLocationSource::None;
253 on_changed();
254 }
255 }
256
257 if (new_remote.partial) {
258 set_partial_remote_location(*new_remote.partial, new_remote.ready_size);
259 } else {
260 delete_partial_remote_location();
261 }
262 }
delete_partial_remote_location()263 void FileNode::delete_partial_remote_location() {
264 if (remote_.partial) {
265 VLOG(update_file) << "File " << main_file_id_ << " has lost partial remote location";
266 remote_.partial.reset();
267 on_changed();
268 }
269 }
270
set_partial_remote_location(PartialRemoteFileLocation remote,int64 ready_size)271 void FileNode::set_partial_remote_location(PartialRemoteFileLocation remote, int64 ready_size) {
272 if (remote_.is_full_alive) {
273 VLOG(update_file) << "File " << main_file_id_ << " remote is still alive, so there is NO reason to update partial";
274 return;
275 }
276 if (remote_.ready_size != ready_size) {
277 VLOG(update_file) << "File " << main_file_id_ << " has changed remote ready size from " << remote_.ready_size
278 << " to " << ready_size;
279 remote_.ready_size = ready_size;
280 on_info_changed();
281 }
282 if (remote_.partial && *remote_.partial == remote) {
283 VLOG(update_file) << "Partial location of " << main_file_id_ << " is NOT changed";
284 return;
285 }
286 if (!remote_.partial && remote.ready_part_count_ == 0) {
287 // empty partial remote is equal to empty remote
288 VLOG(update_file) << "Partial location of " << main_file_id_
289 << " is still empty, so there is NO reason to update it";
290 return;
291 }
292
293 VLOG(update_file) << "File " << main_file_id_ << " partial location has changed to " << remote;
294 remote_.partial = make_unique<PartialRemoteFileLocation>(std::move(remote));
295 on_changed();
296 }
297
delete_file_reference(Slice file_reference)298 bool FileNode::delete_file_reference(Slice file_reference) {
299 if (!remote_.full) {
300 VLOG(file_references) << "Can't delete file reference, because there is no remote location";
301 return false;
302 }
303
304 if (!remote_.full.value().delete_file_reference(file_reference)) {
305 VLOG(file_references) << "Can't delete unmatching file reference " << format::escaped(file_reference) << ", have "
306 << format::escaped(remote_.full.value().get_file_reference());
307 return false;
308 }
309
310 VLOG(file_references) << "Do delete file reference of main file " << main_file_id_;
311 upload_was_update_file_reference_ = false;
312 download_was_update_file_reference_ = false;
313 on_pmc_changed();
314 return true;
315 }
316
set_generate_location(unique_ptr<FullGenerateFileLocation> && generate)317 void FileNode::set_generate_location(unique_ptr<FullGenerateFileLocation> &&generate) {
318 bool is_changed = generate_ == nullptr ? generate != nullptr : generate == nullptr || *generate_ != *generate;
319 if (is_changed) {
320 generate_ = std::move(generate);
321 on_pmc_changed();
322 }
323 }
324
set_size(int64 size)325 void FileNode::set_size(int64 size) {
326 if (size_ != size) {
327 VLOG(update_file) << "File " << main_file_id_ << " has changed size to " << size;
328 size_ = size;
329 on_changed();
330 }
331 }
332
set_expected_size(int64 expected_size)333 void FileNode::set_expected_size(int64 expected_size) {
334 if (expected_size_ != expected_size) {
335 VLOG(update_file) << "File " << main_file_id_ << " has changed expected size to " << expected_size;
336 expected_size_ = expected_size;
337 on_changed();
338 }
339 }
340
set_remote_name(string remote_name)341 void FileNode::set_remote_name(string remote_name) {
342 if (remote_name_ != remote_name) {
343 remote_name_ = std::move(remote_name);
344 on_pmc_changed();
345 }
346 }
347
set_url(string url)348 void FileNode::set_url(string url) {
349 if (url_ != url) {
350 VLOG(update_file) << "File " << main_file_id_ << " has changed URL to " << url;
351 url_ = std::move(url);
352 on_changed();
353 }
354 }
355
set_owner_dialog_id(DialogId owner_id)356 void FileNode::set_owner_dialog_id(DialogId owner_id) {
357 if (owner_dialog_id_ != owner_id) {
358 owner_dialog_id_ = owner_id;
359 on_pmc_changed();
360 }
361 }
362
set_encryption_key(FileEncryptionKey key)363 void FileNode::set_encryption_key(FileEncryptionKey key) {
364 if (encryption_key_ != key) {
365 encryption_key_ = std::move(key);
366 on_pmc_changed();
367 }
368 }
369
set_upload_pause(FileId upload_pause)370 void FileNode::set_upload_pause(FileId upload_pause) {
371 if (upload_pause_ != upload_pause) {
372 LOG(INFO) << "Change file " << main_file_id_ << " upload_pause from " << upload_pause_ << " to " << upload_pause;
373 upload_pause_ = upload_pause;
374 }
375 }
376
set_download_priority(int8 priority)377 void FileNode::set_download_priority(int8 priority) {
378 if ((download_priority_ == 0) != (priority == 0)) {
379 VLOG(update_file) << "File " << main_file_id_ << " has changed download priority to " << priority;
380 on_info_changed();
381 }
382 download_priority_ = priority;
383 }
384
set_upload_priority(int8 priority)385 void FileNode::set_upload_priority(int8 priority) {
386 if (!remote_.is_full_alive && (upload_priority_ == 0) != (priority == 0)) {
387 VLOG(update_file) << "File " << main_file_id_ << " has changed upload priority to " << priority;
388 on_info_changed();
389 }
390 upload_priority_ = priority;
391 }
392
set_generate_priority(int8 download_priority,int8 upload_priority)393 void FileNode::set_generate_priority(int8 download_priority, int8 upload_priority) {
394 if ((generate_download_priority_ == 0) != (download_priority == 0) ||
395 (generate_upload_priority_ == 0) != (upload_priority == 0)) {
396 VLOG(update_file) << "File " << main_file_id_ << " has changed generate priority to " << download_priority << "/"
397 << upload_priority;
398 on_info_changed();
399 }
400 generate_priority_ = max(download_priority, upload_priority);
401 generate_download_priority_ = download_priority;
402 generate_upload_priority_ = upload_priority;
403 }
404
on_changed()405 void FileNode::on_changed() {
406 on_pmc_changed();
407 on_info_changed();
408 }
409
on_info_changed()410 void FileNode::on_info_changed() {
411 info_changed_flag_ = true;
412 }
413
on_pmc_changed()414 void FileNode::on_pmc_changed() {
415 pmc_changed_flag_ = true;
416 }
417
need_info_flush() const418 bool FileNode::need_info_flush() const {
419 return info_changed_flag_;
420 }
421
need_pmc_flush() const422 bool FileNode::need_pmc_flush() const {
423 if (!pmc_changed_flag_) {
424 return false;
425 }
426
427 // already in pmc
428 if (pmc_id_.is_valid()) {
429 return true;
430 }
431
432 // We must save encryption key
433 if (!encryption_key_.empty()) {
434 // && remote_.type() != RemoteFileLocation::Type::Empty
435 return true;
436 }
437
438 bool has_generate_location = generate_ != nullptr;
439 // Do not save "#file_id#" conversion.
440 if (has_generate_location && begins_with(generate_->conversion_, "#file_id#")) {
441 has_generate_location = false;
442 }
443
444 if (remote_.full/* &&
445 (has_generate_location || local_.type() != LocalFileLocation::Type::Empty)*/) {
446 // we need to always save file sources
447 return true;
448 }
449 if (local_.type() == LocalFileLocation::Type::Full && (has_generate_location || remote_.full || remote_.partial)) {
450 return true;
451 }
452
453 // TODO: Generate location with constant conversion
454
455 return false;
456 }
457
on_pmc_flushed()458 void FileNode::on_pmc_flushed() {
459 pmc_changed_flag_ = false;
460 }
on_info_flushed()461 void FileNode::on_info_flushed() {
462 info_changed_flag_ = false;
463 }
464
suggested_path() const465 string FileNode::suggested_path() const {
466 if (!remote_name_.empty()) {
467 return remote_name_;
468 }
469 if (!url_.empty()) {
470 auto file_name = get_url_file_name(url_);
471 if (!file_name.empty()) {
472 return file_name;
473 }
474 }
475 if (generate_ != nullptr) {
476 if (!generate_->original_path_.empty()) {
477 return generate_->original_path_;
478 }
479 }
480 return local_.file_name().str();
481 }
482
483 /*** FileView ***/
has_local_location() const484 bool FileView::has_local_location() const {
485 return node_->local_.type() == LocalFileLocation::Type::Full;
486 }
487
local_location() const488 const FullLocalFileLocation &FileView::local_location() const {
489 CHECK(has_local_location());
490 return node_->local_.full();
491 }
492
has_remote_location() const493 bool FileView::has_remote_location() const {
494 return static_cast<bool>(node_->remote_.full);
495 }
496
has_alive_remote_location() const497 bool FileView::has_alive_remote_location() const {
498 return node_->remote_.is_full_alive;
499 }
500
has_active_upload_remote_location() const501 bool FileView::has_active_upload_remote_location() const {
502 if (!has_remote_location()) {
503 return false;
504 }
505 if (!has_alive_remote_location()) {
506 return false;
507 }
508 if (main_remote_location().is_encrypted_any()) {
509 return true;
510 }
511 return main_remote_location().has_file_reference();
512 }
513
has_active_download_remote_location() const514 bool FileView::has_active_download_remote_location() const {
515 if (!has_remote_location()) {
516 return false;
517 }
518 if (remote_location().is_encrypted_any()) {
519 return true;
520 }
521 return remote_location().has_file_reference();
522 }
523
remote_location() const524 const FullRemoteFileLocation &FileView::remote_location() const {
525 CHECK(has_remote_location());
526 auto *remote = node_.get_remote();
527 if (remote) {
528 return *remote;
529 }
530 return node_->remote_.full.value();
531 }
532
main_remote_location() const533 const FullRemoteFileLocation &FileView::main_remote_location() const {
534 CHECK(has_remote_location());
535 return node_->remote_.full.value();
536 }
537
has_generate_location() const538 bool FileView::has_generate_location() const {
539 return node_->generate_ != nullptr;
540 }
541
generate_location() const542 const FullGenerateFileLocation &FileView::generate_location() const {
543 CHECK(has_generate_location());
544 return *node_->generate_;
545 }
546
size() const547 int64 FileView::size() const {
548 return node_->size_;
549 }
550
get_allocated_local_size() const551 int64 FileView::get_allocated_local_size() const {
552 auto file_path = path();
553 if (file_path.empty()) {
554 return 0;
555 }
556 auto r_stat = stat(file_path);
557 if (r_stat.is_error()) {
558 return 0;
559 }
560 return r_stat.ok().real_size_;
561 }
562
expected_size(bool may_guess) const563 int64 FileView::expected_size(bool may_guess) const {
564 if (node_->size_ != 0) {
565 return node_->size_;
566 }
567 int64 current_size = local_total_size(); // TODO: this is not the best approximation
568 if (node_->expected_size_ != 0) {
569 return max(current_size, node_->expected_size_);
570 }
571 if (may_guess && node_->local_.type() == LocalFileLocation::Type::Partial) {
572 current_size *= 3;
573 }
574 return current_size;
575 }
576
is_downloading() const577 bool FileView::is_downloading() const {
578 return node_->download_priority_ != 0 || node_->generate_download_priority_ != 0;
579 }
580
download_offset() const581 int64 FileView::download_offset() const {
582 return node_->download_offset_;
583 }
584
downloaded_prefix(int64 offset) const585 int64 FileView::downloaded_prefix(int64 offset) const {
586 switch (node_->local_.type()) {
587 case LocalFileLocation::Type::Empty:
588 return 0;
589 case LocalFileLocation::Type::Full:
590 if (offset < node_->size_) {
591 return node_->size_ - offset;
592 }
593 return 0;
594 case LocalFileLocation::Type::Partial:
595 if (is_encrypted_secure()) {
596 // File is not decrypted and verified yet
597 return 0;
598 }
599 return Bitmask(Bitmask::Decode{}, node_->local_.partial().ready_bitmask_)
600 .get_ready_prefix_size(offset, node_->local_.partial().part_size_, node_->size_);
601 default:
602 UNREACHABLE();
603 return 0;
604 }
605 }
606
local_prefix_size() const607 int64 FileView::local_prefix_size() const {
608 switch (node_->local_.type()) {
609 case LocalFileLocation::Type::Full:
610 return node_->download_offset_ <= node_->size_ ? node_->size_ - node_->download_offset_ : 0;
611 case LocalFileLocation::Type::Partial: {
612 if (is_encrypted_secure()) {
613 // File is not decrypted and verified yet
614 return 0;
615 }
616 return node_->local_ready_prefix_size_;
617 }
618 default:
619 return 0;
620 }
621 }
local_total_size() const622 int64 FileView::local_total_size() const {
623 switch (node_->local_.type()) {
624 case LocalFileLocation::Type::Empty:
625 return 0;
626 case LocalFileLocation::Type::Full:
627 return node_->size_;
628 case LocalFileLocation::Type::Partial:
629 VLOG(update_file) << "Have local_ready_prefix_size = " << node_->local_ready_prefix_size_
630 << " and local_ready_size = " << node_->local_ready_size_;
631 return max(node_->local_ready_prefix_size_, node_->local_ready_size_);
632 default:
633 UNREACHABLE();
634 return 0;
635 }
636 }
637
is_uploading() const638 bool FileView::is_uploading() const {
639 return node_->upload_priority_ != 0 || node_->generate_upload_priority_ != 0;
640 }
641
remote_size() const642 int64 FileView::remote_size() const {
643 if (node_->remote_.is_full_alive) {
644 return node_->size_;
645 }
646 if (node_->remote_.partial) {
647 auto part_size = static_cast<int64>(node_->remote_.partial->part_size_);
648 auto ready_part_count = node_->remote_.partial->ready_part_count_;
649 auto remote_ready_size = node_->remote_.ready_size;
650 VLOG(update_file) << "Have part_size = " << part_size << ", remote_ready_part_count = " << ready_part_count
651 << ", remote_ready_size = " << remote_ready_size << ", size = " << size();
652 auto res = max(part_size * ready_part_count, remote_ready_size);
653 if (size() != 0 && size() < res) {
654 res = size();
655 }
656 return res;
657 }
658 return node_->remote_.ready_size; //???
659 }
660
path() const661 string FileView::path() const {
662 switch (node_->local_.type()) {
663 case LocalFileLocation::Type::Full:
664 return node_->local_.full().path_;
665 case LocalFileLocation::Type::Partial:
666 return node_->local_.partial().path_;
667 default:
668 return "";
669 }
670 }
671
has_url() const672 bool FileView::has_url() const {
673 return !node_->url_.empty();
674 }
675
url() const676 const string &FileView::url() const {
677 return node_->url_;
678 }
679
remote_name() const680 const string &FileView::remote_name() const {
681 return node_->remote_name_;
682 }
683
suggested_path() const684 string FileView::suggested_path() const {
685 return node_->suggested_path();
686 }
687
owner_dialog_id() const688 DialogId FileView::owner_dialog_id() const {
689 return node_->owner_dialog_id_;
690 }
691
get_by_hash() const692 bool FileView::get_by_hash() const {
693 return node_->get_by_hash_;
694 }
695
FileView(ConstFileNodePtr node)696 FileView::FileView(ConstFileNodePtr node) : node_(node) {
697 }
698
empty() const699 bool FileView::empty() const {
700 return !node_;
701 }
702
can_download_from_server() const703 bool FileView::can_download_from_server() const {
704 if (!has_remote_location()) {
705 return false;
706 }
707 if (remote_location().file_type_ == FileType::Encrypted && encryption_key().empty()) {
708 return false;
709 }
710 if (remote_location().is_web()) {
711 return true;
712 }
713 if (remote_location().get_dc_id().is_empty()) {
714 return false;
715 }
716 if (!remote_location().is_encrypted_any() && !remote_location().has_file_reference() &&
717 ((node_->download_id_ == 0 && node_->download_was_update_file_reference_) || !node_->remote_.is_full_alive)) {
718 return false;
719 }
720 return true;
721 }
722
can_generate() const723 bool FileView::can_generate() const {
724 return has_generate_location();
725 }
726
can_delete() const727 bool FileView::can_delete() const {
728 if (has_local_location()) {
729 return begins_with(local_location().path_, get_files_dir(get_type()));
730 }
731 return node_->local_.type() == LocalFileLocation::Type::Partial;
732 }
733
get_unique_id(const FullGenerateFileLocation & location)734 string FileView::get_unique_id(const FullGenerateFileLocation &location) {
735 return base64url_encode(zero_encode('\xff' + serialize(location)));
736 }
737
get_unique_id(const FullRemoteFileLocation & location)738 string FileView::get_unique_id(const FullRemoteFileLocation &location) {
739 return base64url_encode(zero_encode(serialize(location.as_unique())));
740 }
741
get_persistent_id(const FullGenerateFileLocation & location)742 string FileView::get_persistent_id(const FullGenerateFileLocation &location) {
743 auto binary = serialize(location);
744
745 binary = zero_encode(binary);
746 binary.push_back(FileNode::PERSISTENT_ID_VERSION_MAP);
747 return base64url_encode(binary);
748 }
749
get_persistent_id(const FullRemoteFileLocation & location)750 string FileView::get_persistent_id(const FullRemoteFileLocation &location) {
751 auto binary = serialize(location);
752
753 binary = zero_encode(binary);
754 binary.push_back(static_cast<char>(narrow_cast<uint8>(Version::Next) - 1));
755 binary.push_back(FileNode::PERSISTENT_ID_VERSION);
756 return base64url_encode(binary);
757 }
758
get_persistent_file_id() const759 string FileView::get_persistent_file_id() const {
760 if (!empty()) {
761 if (has_alive_remote_location()) {
762 return get_persistent_id(remote_location());
763 } else if (has_url()) {
764 return url();
765 } else if (has_generate_location() && begins_with(generate_location().conversion_, "#map#")) {
766 return get_persistent_id(generate_location());
767 }
768 }
769 return string();
770 }
771
get_unique_file_id() const772 string FileView::get_unique_file_id() const {
773 if (!empty()) {
774 if (has_alive_remote_location()) {
775 if (!remote_location().is_web()) {
776 return get_unique_id(remote_location());
777 }
778 } else if (has_generate_location() && begins_with(generate_location().conversion_, "#map#")) {
779 return get_unique_id(generate_location());
780 }
781 }
782 return string();
783 }
784
785 /*** FileManager ***/
786 static int merge_choose_remote_location(const FullRemoteFileLocation &x, FileLocationSource x_source,
787 const FullRemoteFileLocation &y, FileLocationSource y_source);
788
789 namespace {
prepare_path_for_pmc(FileType file_type,string & path)790 void prepare_path_for_pmc(FileType file_type, string &path) {
791 path = PathView::relative(path, get_files_base_dir(file_type)).str();
792 }
793 } // namespace
794
FileManager(unique_ptr<Context> context)795 FileManager::FileManager(unique_ptr<Context> context) : context_(std::move(context)) {
796 if (G()->parameters().use_file_db) {
797 file_db_ = G()->td_db()->get_file_db_shared();
798 }
799
800 parent_ = context_->create_reference();
801 next_file_id();
802 next_file_node_id();
803
804 std::unordered_set<string> dir_paths;
805 for (int32 i = 0; i < MAX_FILE_TYPE; i++) {
806 dir_paths.insert(get_files_dir(static_cast<FileType>(i)));
807 }
808 // add both temp dirs
809 dir_paths.insert(get_files_temp_dir(FileType::Encrypted));
810 dir_paths.insert(get_files_temp_dir(FileType::Video));
811
812 for (const auto &path : dir_paths) {
813 auto status = mkdir(path, 0750);
814 if (status.is_error()) {
815 auto r_stat = stat(path);
816 if (r_stat.is_ok() && r_stat.ok().is_dir_) {
817 LOG(ERROR) << "Creation of directory \"" << path << "\" failed with " << status << ", but directory exists";
818 } else {
819 LOG(ERROR) << "Creation of directory \"" << path << "\" failed with " << status;
820 }
821 }
822 #if TD_ANDROID
823 FileFd::open(path + ".nomedia", FileFd::Create | FileFd::Read).ignore();
824 #endif
825 };
826
827 G()->td_db()->with_db_path([bad_paths = &bad_paths_](CSlice path) { bad_paths->insert(path.str()); });
828 }
829
init_actor()830 void FileManager::init_actor() {
831 file_load_manager_ = create_actor_on_scheduler<FileLoadManager>("FileLoadManager", G()->get_slow_net_scheduler_id(),
832 actor_shared(this), context_->create_reference());
833 file_generate_manager_ = create_actor_on_scheduler<FileGenerateManager>(
834 "FileGenerateManager", G()->get_slow_net_scheduler_id(), context_->create_reference());
835 }
836
~FileManager()837 FileManager::~FileManager() {
838 }
839
fix_file_extension(Slice file_name,Slice file_type,Slice file_extension)840 string FileManager::fix_file_extension(Slice file_name, Slice file_type, Slice file_extension) {
841 return (file_name.empty() ? file_type : file_name).str() + "." + file_extension.str();
842 }
843
get_file_name(FileType file_type,Slice path)844 string FileManager::get_file_name(FileType file_type, Slice path) {
845 PathView path_view(path);
846 auto file_name = path_view.file_name();
847 auto extension = path_view.extension();
848 switch (file_type) {
849 case FileType::Thumbnail:
850 if (extension != "jpg" && extension != "jpeg" && extension != "webp") {
851 return fix_file_extension(file_name, "thumbnail", "jpg");
852 }
853 break;
854 case FileType::ProfilePhoto:
855 case FileType::Photo:
856 if (extension != "jpg" && extension != "jpeg" && extension != "gif" && extension != "png" && extension != "tif" &&
857 extension != "bmp") {
858 return fix_file_extension(file_name, "photo", "jpg");
859 }
860 break;
861 case FileType::VoiceNote:
862 if (extension != "ogg" && extension != "oga" && extension != "mp3" && extension != "mpeg3" &&
863 extension != "m4a" && extension != "opus") {
864 return fix_file_extension(file_name, "voice", "oga");
865 }
866 break;
867 case FileType::Video:
868 case FileType::VideoNote:
869 if (extension != "mov" && extension != "3gp" && extension != "mpeg4" && extension != "mp4") {
870 return fix_file_extension(file_name, "video", "mp4");
871 }
872 break;
873 case FileType::Audio:
874 if (extension != "ogg" && extension != "oga" && extension != "mp3" && extension != "mpeg3" &&
875 extension != "m4a") {
876 return fix_file_extension(file_name, "audio", "mp3");
877 }
878 break;
879 case FileType::Wallpaper:
880 case FileType::Background:
881 if (extension != "jpg" && extension != "jpeg" && extension != "png") {
882 return fix_file_extension(file_name, "wallpaper", "jpg");
883 }
884 break;
885 case FileType::Sticker:
886 if (extension != "webp" && extension != "tgs") {
887 return fix_file_extension(file_name, "sticker", "webp");
888 }
889 break;
890 case FileType::Document:
891 case FileType::Animation:
892 case FileType::Encrypted:
893 case FileType::Temp:
894 case FileType::EncryptedThumbnail:
895 case FileType::Secure:
896 case FileType::SecureRaw:
897 case FileType::DocumentAsFile:
898 break;
899 default:
900 UNREACHABLE();
901 }
902 return file_name.str();
903 }
904
are_modification_times_equal(int64 old_mtime,int64 new_mtime)905 bool FileManager::are_modification_times_equal(int64 old_mtime, int64 new_mtime) {
906 if (old_mtime == new_mtime) {
907 return true;
908 }
909 if (old_mtime < new_mtime) {
910 return false;
911 }
912 if (old_mtime - new_mtime == 1000000000 && old_mtime % 1000000000 == 0 && new_mtime % 2000000000 == 0) {
913 // FAT32 has 2 seconds mtime resolution, but file system sometimes reports odd modification time
914 return true;
915 }
916 return false;
917 }
918
check_local_location(FullLocalFileLocation & location,int64 & size,bool skip_file_size_checks)919 Status FileManager::check_local_location(FullLocalFileLocation &location, int64 &size, bool skip_file_size_checks) {
920 constexpr int64 MAX_THUMBNAIL_SIZE = 200 * (1 << 10) - 1 /* 200 KB - 1 B */;
921 constexpr int64 MAX_PHOTO_SIZE = 10 * (1 << 20) /* 10 MB */;
922 constexpr int64 DEFAULT_VIDEO_NOTE_SIZE_MAX = 12 * (1 << 20) /* 12 MB */;
923
924 if (location.path_.empty()) {
925 return Status::Error(400, "File must have non-empty path");
926 }
927 auto r_path = realpath(location.path_, true);
928 if (r_path.is_error()) {
929 return Status::Error(400, "Can't find real file path");
930 }
931 location.path_ = r_path.move_as_ok();
932 if (bad_paths_.count(location.path_) != 0) {
933 return Status::Error(400, "Sending of internal database files is forbidden");
934 }
935 auto r_stat = stat(location.path_);
936 if (r_stat.is_error()) {
937 return Status::Error(400, "Can't get stat about the file");
938 }
939 auto stat = r_stat.move_as_ok();
940 if (!stat.is_reg_) {
941 return Status::Error(400, "File must be a regular file");
942 }
943 if (stat.size_ < 0) {
944 // TODO is it possible?
945 return Status::Error(400, "File is too big");
946 }
947 if (stat.size_ == 0) {
948 return Status::Error(400, "File must be non-empty");
949 }
950
951 if (size == 0) {
952 size = stat.size_;
953 }
954 if (location.mtime_nsec_ == 0) {
955 VLOG(file_loader) << "Set file \"" << location.path_ << "\" modification time to " << stat.mtime_nsec_;
956 location.mtime_nsec_ = stat.mtime_nsec_;
957 } else if (!are_modification_times_equal(location.mtime_nsec_, stat.mtime_nsec_)) {
958 VLOG(file_loader) << "File \"" << location.path_ << "\" was modified: old mtime = " << location.mtime_nsec_
959 << ", new mtime = " << stat.mtime_nsec_;
960 return Status::Error(400, PSLICE() << "File \"" << location.path_ << "\" was modified");
961 }
962 if (skip_file_size_checks) {
963 return Status::OK();
964 }
965 if ((location.file_type_ == FileType::Thumbnail || location.file_type_ == FileType::EncryptedThumbnail) &&
966 size > MAX_THUMBNAIL_SIZE && !begins_with(PathView(location.path_).file_name(), "map")) {
967 return Status::Error(400, PSLICE() << "File \"" << location.path_ << "\" is too big for a thumbnail "
968 << tag("size", format::as_size(size)));
969 }
970 if (size > MAX_FILE_SIZE) {
971 return Status::Error(400, PSLICE() << "File \"" << location.path_ << "\" of size " << size << " bytes is too big");
972 }
973 if (location.file_type_ == FileType::Photo && size > MAX_PHOTO_SIZE) {
974 return Status::Error(
975 400, PSLICE() << "File \"" << location.path_ << "\" of size " << size << " bytes is too big for a photo");
976 }
977 if (location.file_type_ == FileType::VideoNote &&
978 size > G()->shared_config().get_option_integer("video_note_size_max", DEFAULT_VIDEO_NOTE_SIZE_MAX)) {
979 return Status::Error(
980 400, PSLICE() << "File \"" << location.path_ << "\" of size " << size << " bytes is too big for a video note");
981 }
982 return Status::OK();
983 }
984
check_partial_local_location(const PartialLocalFileLocation & location)985 static Status check_partial_local_location(const PartialLocalFileLocation &location) {
986 TRY_RESULT(stat, stat(location.path_));
987 if (!stat.is_reg_) {
988 if (stat.is_dir_) {
989 return Status::Error(PSLICE() << "Can't use directory \"" << location.path_ << "\" as a file path");
990 }
991 return Status::Error("File must be a regular file");
992 }
993 // can't check mtime. Hope nobody will mess with this file in our temporary dir.
994 return Status::OK();
995 }
996
check_local_location(FileNodePtr node)997 Status FileManager::check_local_location(FileNodePtr node) {
998 Status status;
999 if (node->local_.type() == LocalFileLocation::Type::Full) {
1000 status = check_local_location(node->local_.full(), node->size_, false);
1001 } else if (node->local_.type() == LocalFileLocation::Type::Partial) {
1002 status = check_partial_local_location(node->local_.partial());
1003 }
1004 if (status.is_error()) {
1005 node->drop_local_location();
1006 try_flush_node(node, "check_local_location");
1007 }
1008 return status;
1009 }
1010
try_fix_partial_local_location(FileNodePtr node)1011 bool FileManager::try_fix_partial_local_location(FileNodePtr node) {
1012 LOG(INFO) << "Trying to fix partial local location";
1013 if (node->local_.type() != LocalFileLocation::Type::Partial) {
1014 LOG(INFO) << " failed - not a partial location";
1015 return false;
1016 }
1017 auto partial = node->local_.partial();
1018 if (!partial.iv_.empty()) {
1019 // can't recalc iv_
1020 LOG(INFO) << " failed - partial location has nonempty iv";
1021 return false;
1022 }
1023 if (partial.part_size_ >= 512 * (1 << 10)) {
1024 LOG(INFO) << " failed - too big part_size already: " << partial.part_size_;
1025 return false;
1026 }
1027 auto old_part_size = partial.part_size_;
1028 int new_part_size = 512 * (1 << 10);
1029 auto k = new_part_size / old_part_size;
1030 Bitmask mask(Bitmask::Decode(), partial.ready_bitmask_);
1031 auto new_mask = mask.compress(k);
1032
1033 partial.part_size_ = new_part_size;
1034 partial.ready_bitmask_ = new_mask.encode();
1035
1036 auto ready_size = new_mask.get_total_size(partial.part_size_, node->size_);
1037 node->set_local_location(LocalFileLocation(std::move(partial)), ready_size, -1, -1);
1038 LOG(INFO) << " ok: increase part_size " << old_part_size << "->" << new_part_size;
1039 return true;
1040 }
1041
get_file_id_info(FileId file_id)1042 FileManager::FileIdInfo *FileManager::get_file_id_info(FileId file_id) {
1043 CHECK(static_cast<size_t>(file_id.get()) < file_id_info_.size());
1044 return &file_id_info_[file_id.get()];
1045 }
1046
dup_file_id(FileId file_id)1047 FileId FileManager::dup_file_id(FileId file_id) {
1048 int32 file_node_id;
1049 auto *file_node = get_file_node_raw(file_id, &file_node_id);
1050 if (!file_node) {
1051 return FileId();
1052 }
1053 auto result = FileId(create_file_id(file_node_id, file_node).get(), file_id.get_remote());
1054 LOG(INFO) << "Dup file " << file_id << " to " << result;
1055 return result;
1056 }
1057
create_file_id(int32 file_node_id,FileNode * file_node)1058 FileId FileManager::create_file_id(int32 file_node_id, FileNode *file_node) {
1059 auto file_id = next_file_id();
1060 get_file_id_info(file_id)->node_id_ = file_node_id;
1061 file_node->file_ids_.push_back(file_id);
1062 return file_id;
1063 }
1064
try_forget_file_id(FileId file_id)1065 void FileManager::try_forget_file_id(FileId file_id) {
1066 auto *info = get_file_id_info(file_id);
1067 if (info->send_updates_flag_ || info->pin_flag_ || info->sent_file_id_flag_) {
1068 return;
1069 }
1070 auto file_node = get_file_node(file_id);
1071 if (file_node->main_file_id_ == file_id) {
1072 return;
1073 }
1074
1075 LOG(DEBUG) << "Forget file " << file_id;
1076 bool is_removed = td::remove(file_node->file_ids_, file_id);
1077 CHECK(is_removed);
1078 *info = FileIdInfo();
1079 empty_file_ids_.push_back(file_id.get());
1080 }
1081
register_empty(FileType type)1082 FileId FileManager::register_empty(FileType type) {
1083 return register_local(FullLocalFileLocation(type, "", 0), DialogId(), 0, false, true).ok();
1084 }
1085
on_file_unlink(const FullLocalFileLocation & location)1086 void FileManager::on_file_unlink(const FullLocalFileLocation &location) {
1087 // TODO: remove file from the database too
1088 auto it = local_location_to_file_id_.find(location);
1089 if (it == local_location_to_file_id_.end()) {
1090 return;
1091 }
1092 auto file_id = it->second;
1093 auto file_node = get_sync_file_node(file_id);
1094 CHECK(file_node);
1095 file_node->drop_local_location();
1096 try_flush_node_info(file_node, "on_file_unlink");
1097 }
1098
register_local(FullLocalFileLocation location,DialogId owner_dialog_id,int64 size,bool get_by_hash,bool force,bool skip_file_size_checks)1099 Result<FileId> FileManager::register_local(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size,
1100 bool get_by_hash, bool force, bool skip_file_size_checks) {
1101 // TODO: use get_by_hash
1102 FileData data;
1103 data.local_ = LocalFileLocation(std::move(location));
1104 data.owner_dialog_id_ = owner_dialog_id;
1105 data.size_ = size;
1106 return register_file(std::move(data), FileLocationSource::None /*won't be used*/, "register_local", force,
1107 skip_file_size_checks);
1108 }
1109
register_remote(FullRemoteFileLocation location,FileLocationSource file_location_source,DialogId owner_dialog_id,int64 size,int64 expected_size,string remote_name)1110 FileId FileManager::register_remote(FullRemoteFileLocation location, FileLocationSource file_location_source,
1111 DialogId owner_dialog_id, int64 size, int64 expected_size, string remote_name) {
1112 FileData data;
1113 auto url = location.get_url();
1114 data.remote_ = RemoteFileLocation(std::move(location));
1115 data.owner_dialog_id_ = owner_dialog_id;
1116 data.size_ = size;
1117 data.expected_size_ = expected_size;
1118 data.remote_name_ = std::move(remote_name);
1119
1120 auto file_id = register_file(std::move(data), file_location_source, "register_remote", false).move_as_ok();
1121 if (!url.empty()) {
1122 auto file_node = get_file_node(file_id);
1123 CHECK(file_node);
1124 file_node->set_url(url);
1125 }
1126 return file_id;
1127 }
1128
register_url(string url,FileType file_type,FileLocationSource file_location_source,DialogId owner_dialog_id)1129 FileId FileManager::register_url(string url, FileType file_type, FileLocationSource file_location_source,
1130 DialogId owner_dialog_id) {
1131 auto file_id = register_generate(file_type, file_location_source, url, "#url#", owner_dialog_id, 0).ok();
1132 auto file_node = get_file_node(file_id);
1133 CHECK(file_node);
1134 file_node->set_url(url);
1135 return file_id;
1136 }
1137
register_generate(FileType file_type,FileLocationSource file_location_source,string original_path,string conversion,DialogId owner_dialog_id,int64 expected_size)1138 Result<FileId> FileManager::register_generate(FileType file_type, FileLocationSource file_location_source,
1139 string original_path, string conversion, DialogId owner_dialog_id,
1140 int64 expected_size) {
1141 // add #mtime# into conversion
1142 if (!original_path.empty() && conversion[0] != '#' && PathView(original_path).is_absolute()) {
1143 auto file_paths = log_interface->get_file_paths();
1144 if (!td::contains(file_paths, original_path)) {
1145 auto r_stat = stat(original_path);
1146 uint64 mtime = r_stat.is_ok() ? r_stat.ok().mtime_nsec_ : 0;
1147 conversion = PSTRING() << "#mtime#" << lpad0(to_string(mtime), 20) << '#' << conversion;
1148 }
1149 }
1150
1151 FileData data;
1152 data.generate_ =
1153 td::make_unique<FullGenerateFileLocation>(file_type, std::move(original_path), std::move(conversion));
1154 data.owner_dialog_id_ = owner_dialog_id;
1155 data.expected_size_ = expected_size;
1156 return register_file(std::move(data), file_location_source, "register_generate", false);
1157 }
1158
register_file(FileData && data,FileLocationSource file_location_source,const char * source,bool force,bool skip_file_size_checks)1159 Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource file_location_source, const char *source,
1160 bool force, bool skip_file_size_checks) {
1161 bool has_remote = data.remote_.type() == RemoteFileLocation::Type::Full;
1162 bool has_generate = data.generate_ != nullptr;
1163 if (data.local_.type() == LocalFileLocation::Type::Full && !force) {
1164 if (file_location_source == FileLocationSource::FromBinlog ||
1165 file_location_source == FileLocationSource::FromDatabase) {
1166 PathView path_view(data.local_.full().path_);
1167 if (path_view.is_relative()) {
1168 data.local_.full().path_ = PSTRING()
1169 << get_files_base_dir(data.local_.full().file_type_) << data.local_.full().path_;
1170 }
1171 }
1172
1173 auto status = check_local_location(data.local_.full(), data.size_, skip_file_size_checks);
1174 if (status.is_error()) {
1175 LOG(INFO) << "Invalid " << data.local_.full() << ": " << status << " from " << source;
1176 data.local_ = LocalFileLocation();
1177 if (data.remote_.type() == RemoteFileLocation::Type::Partial) {
1178 data.remote_ = {};
1179 }
1180
1181 if (!has_remote && !has_generate) {
1182 return std::move(status);
1183 }
1184 }
1185 }
1186 bool has_local = data.local_.type() == LocalFileLocation::Type::Full;
1187 bool has_location = has_local || has_remote || has_generate;
1188 if (!has_location) {
1189 return Status::Error(400, "No location");
1190 }
1191
1192 FileId file_id = next_file_id();
1193
1194 LOG(INFO) << "Register file data " << data << " as " << file_id << " from " << source;
1195 // create FileNode
1196 auto file_node_id = next_file_node_id();
1197 auto &node = file_nodes_[file_node_id];
1198 node = td::make_unique<FileNode>(std::move(data.local_), NewRemoteFileLocation(data.remote_, file_location_source),
1199 std::move(data.generate_), data.size_, data.expected_size_,
1200 std::move(data.remote_name_), std::move(data.url_), data.owner_dialog_id_,
1201 std::move(data.encryption_key_), file_id, static_cast<int8>(has_remote));
1202 node->pmc_id_ = FileDbId(data.pmc_id_);
1203 get_file_id_info(file_id)->node_id_ = file_node_id;
1204 node->file_ids_.push_back(file_id);
1205
1206 FileView file_view(get_file_node(file_id));
1207
1208 std::vector<FileId> to_merge;
1209 auto register_location = [&](const auto &location, auto &mp) {
1210 auto &other_id = mp[location];
1211 if (other_id.empty()) {
1212 other_id = file_id;
1213 get_file_id_info(file_id)->pin_flag_ = true;
1214 return true;
1215 } else {
1216 to_merge.push_back(other_id);
1217 return false;
1218 }
1219 };
1220 bool new_remote = false;
1221 int32 remote_key = 0;
1222 if (file_view.has_remote_location()) {
1223 RemoteInfo info{file_view.remote_location(), file_location_source, file_id};
1224 remote_key = remote_location_info_.add(info);
1225 auto &stored_info = remote_location_info_.get(remote_key);
1226 if (stored_info.file_id_ == file_id) {
1227 get_file_id_info(file_id)->pin_flag_ = true;
1228 new_remote = true;
1229 } else {
1230 to_merge.push_back(stored_info.file_id_);
1231 if (merge_choose_remote_location(file_view.remote_location(), file_location_source, stored_info.remote_,
1232 stored_info.file_location_source_) == 0) {
1233 stored_info.remote_ = file_view.remote_location();
1234 stored_info.file_location_source_ = file_location_source;
1235 }
1236 }
1237 }
1238 bool new_local = false;
1239 if (file_view.has_local_location()) {
1240 new_local = register_location(file_view.local_location(), local_location_to_file_id_);
1241 }
1242 bool new_generate = false;
1243 if (file_view.has_generate_location()) {
1244 new_generate = register_location(file_view.generate_location(), generate_location_to_file_id_);
1245 }
1246 td::unique(to_merge);
1247
1248 int new_cnt = new_remote + new_local + new_generate;
1249 if (data.pmc_id_ == 0 && file_db_ && new_cnt > 0) {
1250 node->need_load_from_pmc_ = true;
1251 }
1252 bool no_sync_merge = to_merge.size() == 1 && new_cnt == 0;
1253 for (auto id : to_merge) {
1254 // may invalidate node
1255 merge(file_id, id, no_sync_merge).ignore();
1256 }
1257
1258 try_flush_node(get_file_node(file_id), "register_file");
1259 auto main_file_id = get_file_node(file_id)->main_file_id_;
1260 try_forget_file_id(file_id);
1261 for (auto file_source_id : data.file_source_ids_) {
1262 VLOG(file_references) << "Loaded " << data.file_source_ids_ << " for file " << main_file_id << " from " << source;
1263 if (file_source_id.is_valid()) {
1264 context_->add_file_source(main_file_id, file_source_id);
1265 }
1266 }
1267 return FileId(main_file_id.get(), remote_key);
1268 }
1269
1270 // 0 -- choose x
1271 // 1 -- choose y
1272 // 2 -- choose any
merge_choose_local_location(const LocalFileLocation & x,const LocalFileLocation & y)1273 static int merge_choose_local_location(const LocalFileLocation &x, const LocalFileLocation &y) {
1274 auto x_type = static_cast<int32>(x.type());
1275 auto y_type = static_cast<int32>(y.type());
1276 if (x_type != y_type) {
1277 return x_type < y_type;
1278 }
1279 return 2;
1280 }
1281
merge_choose_file_source_location(FileLocationSource x,FileLocationSource y)1282 static int merge_choose_file_source_location(FileLocationSource x, FileLocationSource y) {
1283 return static_cast<int>(x) < static_cast<int>(y);
1284 }
1285
merge_choose_remote_location(const FullRemoteFileLocation & x,FileLocationSource x_source,const FullRemoteFileLocation & y,FileLocationSource y_source)1286 static int merge_choose_remote_location(const FullRemoteFileLocation &x, FileLocationSource x_source,
1287 const FullRemoteFileLocation &y, FileLocationSource y_source) {
1288 LOG(INFO) << "Choose between " << x << " from " << x_source << " and " << y << " from " << y_source;
1289 if (x.is_web() != y.is_web()) {
1290 return x.is_web(); // prefer non-web
1291 }
1292 auto x_ref = x.has_file_reference();
1293 auto y_ref = y.has_file_reference();
1294 if (x_ref || y_ref) {
1295 if (x_ref != y_ref) {
1296 return !x_ref;
1297 }
1298 if (x.get_file_reference() != y.get_file_reference()) {
1299 return merge_choose_file_source_location(x_source, y_source);
1300 }
1301 }
1302 if ((x.get_access_hash() != y.get_access_hash() || x.get_source() != y.get_source()) &&
1303 (x_source != y_source || x.is_web() || x.get_id() == y.get_id())) {
1304 return merge_choose_file_source_location(x_source, y_source);
1305 }
1306 return 2;
1307 }
1308
merge_choose_remote_location(const NewRemoteFileLocation & x,const NewRemoteFileLocation & y)1309 static int merge_choose_remote_location(const NewRemoteFileLocation &x, const NewRemoteFileLocation &y) {
1310 if (x.is_full_alive != y.is_full_alive) {
1311 return !x.is_full_alive;
1312 }
1313 if (x.is_full_alive) {
1314 return merge_choose_remote_location(x.full.value(), x.full_source, y.full.value(), y.full_source);
1315 }
1316 if (!x.partial != !y.partial) {
1317 return !x.partial;
1318 }
1319 return 2;
1320 }
1321
merge_choose_generate_location(const unique_ptr<FullGenerateFileLocation> & x,const unique_ptr<FullGenerateFileLocation> & y)1322 static int merge_choose_generate_location(const unique_ptr<FullGenerateFileLocation> &x,
1323 const unique_ptr<FullGenerateFileLocation> &y) {
1324 int x_empty = (x == nullptr);
1325 int y_empty = (y == nullptr);
1326 if (x_empty != y_empty) {
1327 return x_empty ? 1 : 0;
1328 }
1329 if (!x_empty && *x != *y) {
1330 bool x_has_mtime = begins_with(x->conversion_, "#mtime#");
1331 bool y_has_mtime = begins_with(y->conversion_, "#mtime#");
1332 if (x_has_mtime != y_has_mtime) {
1333 return x_has_mtime ? 0 : 1;
1334 }
1335 return x->conversion_ >= y->conversion_
1336 ? 0
1337 : 1; // the bigger conversion, the bigger mtime or at least more stable choise
1338 }
1339 return 2;
1340 }
1341
1342 // -1 -- error
merge_choose_size(int64 x,int64 y)1343 static int merge_choose_size(int64 x, int64 y) {
1344 if (x == 0) {
1345 return 1;
1346 }
1347 if (y == 0) {
1348 return 0;
1349 }
1350 if (x != y) {
1351 return -1;
1352 }
1353 return 2;
1354 }
1355
merge_choose_expected_size(int64 x,int64 y)1356 static int merge_choose_expected_size(int64 x, int64 y) {
1357 if (x == 0) {
1358 return 1;
1359 }
1360 if (y == 0) {
1361 return 0;
1362 }
1363 return 2;
1364 }
1365
merge_choose_name(Slice x,Slice y)1366 static int merge_choose_name(Slice x, Slice y) {
1367 if (x.empty() != y.empty()) {
1368 return x.empty() > y.empty();
1369 }
1370 return 2;
1371 }
1372
merge_choose_owner(DialogId x,DialogId y)1373 static int merge_choose_owner(DialogId x, DialogId y) {
1374 if (x.is_valid() != y.is_valid()) {
1375 return x.is_valid() < y.is_valid();
1376 }
1377 return 2;
1378 }
1379
merge_choose_main_file_id(FileId a,int8 a_priority,FileId b,int8 b_priority)1380 static int merge_choose_main_file_id(FileId a, int8 a_priority, FileId b, int8 b_priority) {
1381 if (a_priority != b_priority) {
1382 return a_priority < b_priority;
1383 }
1384 return a.get() > b.get();
1385 }
1386
merge_choose_encryption_key(const FileEncryptionKey & a,const FileEncryptionKey & b)1387 static int merge_choose_encryption_key(const FileEncryptionKey &a, const FileEncryptionKey &b) {
1388 if (a.empty() != b.empty()) {
1389 return a.empty() > b.empty();
1390 }
1391 if (a != b) {
1392 return -1;
1393 }
1394 return 2;
1395 }
1396
do_cancel_download(FileNodePtr node)1397 void FileManager::do_cancel_download(FileNodePtr node) {
1398 if (node->download_id_ == 0) {
1399 return;
1400 }
1401 send_closure(file_load_manager_, &FileLoadManager::cancel, node->download_id_);
1402 node->download_id_ = 0;
1403 node->is_download_started_ = false;
1404 node->download_was_update_file_reference_ = false;
1405 node->set_download_priority(0);
1406 }
1407
do_cancel_upload(FileNodePtr node)1408 void FileManager::do_cancel_upload(FileNodePtr node) {
1409 if (node->upload_id_ == 0) {
1410 return;
1411 }
1412 send_closure(file_load_manager_, &FileLoadManager::cancel, node->upload_id_);
1413 node->upload_id_ = 0;
1414 node->upload_was_update_file_reference_ = false;
1415 node->set_upload_priority(0);
1416 }
1417
do_cancel_generate(FileNodePtr node)1418 void FileManager::do_cancel_generate(FileNodePtr node) {
1419 if (node->generate_id_ == 0) {
1420 return;
1421 }
1422 send_closure(file_generate_manager_, &FileGenerateManager::cancel, node->generate_id_);
1423 node->generate_id_ = 0;
1424 node->generate_was_update_ = false;
1425 node->set_generate_priority(0, 0);
1426 }
1427
merge(FileId x_file_id,FileId y_file_id,bool no_sync)1428 Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) {
1429 LOG(DEBUG) << "Merge new file " << x_file_id << " and old file " << y_file_id;
1430
1431 if (!x_file_id.is_valid()) {
1432 return Status::Error("First file_id is invalid");
1433 }
1434 FileNodePtr x_node = no_sync ? get_file_node(x_file_id) : get_sync_file_node(x_file_id);
1435 if (!x_node) {
1436 return Status::Error(PSLICE() << "Can't merge files. First identifier is invalid: " << x_file_id << " and "
1437 << y_file_id);
1438 }
1439
1440 if (!y_file_id.is_valid()) {
1441 LOG(DEBUG) << "Old file is invalid";
1442 return x_node->main_file_id_;
1443 }
1444 FileNodePtr y_node = get_file_node(y_file_id);
1445 if (!y_node) {
1446 return Status::Error(PSLICE() << "Can't merge files. Second identifier is invalid: " << x_file_id << " and "
1447 << y_file_id);
1448 }
1449
1450 if (x_file_id == x_node->upload_pause_) {
1451 x_node->set_upload_pause(FileId());
1452 }
1453 if (x_node.get() == y_node.get()) {
1454 LOG(DEBUG) << "Files are already merged";
1455 return x_node->main_file_id_;
1456 }
1457 if (y_file_id == y_node->upload_pause_) {
1458 y_node->set_upload_pause(FileId());
1459 }
1460
1461 if (x_node->remote_.full && y_node->remote_.full && !x_node->remote_.full.value().is_web() &&
1462 !y_node->remote_.full.value().is_web() && y_node->remote_.is_full_alive &&
1463 x_node->remote_.full_source == FileLocationSource::FromServer &&
1464 y_node->remote_.full_source == FileLocationSource::FromServer &&
1465 x_node->remote_.full.value().get_dc_id() != y_node->remote_.full.value().get_dc_id()) {
1466 LOG(ERROR) << "File remote location was changed from " << y_node->remote_.full.value() << " to "
1467 << x_node->remote_.full.value();
1468 }
1469
1470 bool drop_last_successful_force_reupload_time = x_node->last_successful_force_reupload_time_ <= 0 &&
1471 x_node->remote_.full &&
1472 x_node->remote_.full_source == FileLocationSource::FromServer;
1473
1474 auto count_local = [](auto &node) {
1475 return std::accumulate(node->file_ids_.begin(), node->file_ids_.end(), 0,
1476 [](const auto &x, const auto &y) { return x + (y.get_remote() != 0); });
1477 };
1478 auto x_local_file_ids = count_local(x_node);
1479 auto y_local_file_ids = count_local(y_node);
1480 if (x_local_file_ids + y_local_file_ids > 100) {
1481 }
1482
1483 if (y_node->file_ids_.size() >= 100 || x_node->file_ids_.size() >= 100) {
1484 LOG(INFO) << "Merge files with " << x_local_file_ids << '/' << x_node->file_ids_.size() << " and "
1485 << y_local_file_ids << '/' << y_node->file_ids_.size() << " file IDs";
1486 }
1487
1488 FileNodePtr nodes[] = {x_node, y_node, x_node};
1489 FileNodeId node_ids[] = {get_file_id_info(x_file_id)->node_id_, get_file_id_info(y_file_id)->node_id_};
1490 int trusted_by_source = merge_choose_file_source_location(x_node->remote_.full_source, y_node->remote_.full_source);
1491
1492 int local_i = merge_choose_local_location(x_node->local_, y_node->local_);
1493 int remote_i = merge_choose_remote_location(x_node->remote_, y_node->remote_);
1494 int generate_i = merge_choose_generate_location(x_node->generate_, y_node->generate_);
1495 int size_i = merge_choose_size(x_node->size_, y_node->size_);
1496 int expected_size_i = merge_choose_expected_size(x_node->expected_size_, y_node->expected_size_);
1497 int remote_name_i = merge_choose_name(x_node->remote_name_, y_node->remote_name_);
1498 int url_i = merge_choose_name(x_node->url_, y_node->url_);
1499 int owner_i = merge_choose_owner(x_node->owner_dialog_id_, y_node->owner_dialog_id_);
1500 int encryption_key_i = merge_choose_encryption_key(x_node->encryption_key_, y_node->encryption_key_);
1501 int main_file_id_i = merge_choose_main_file_id(x_node->main_file_id_, x_node->main_file_id_priority_,
1502 y_node->main_file_id_, y_node->main_file_id_priority_);
1503
1504 if (size_i == -1) {
1505 return Status::Error(PSLICE() << "Can't merge files. Different size: " << x_node->size_ << " and "
1506 << y_node->size_);
1507 }
1508 if (encryption_key_i == -1) {
1509 if (nodes[remote_i]->remote_.full && nodes[local_i]->local_.type() != LocalFileLocation::Type::Partial) {
1510 LOG(ERROR) << "Different encryption key in files, but lets choose same key as remote location";
1511 encryption_key_i = remote_i;
1512 } else {
1513 return Status::Error("Can't merge files. Different encryption keys");
1514 }
1515 }
1516
1517 // prefer more trusted source
1518 if (remote_name_i == 2) {
1519 remote_name_i = trusted_by_source;
1520 }
1521 if (url_i == 2) {
1522 url_i = trusted_by_source;
1523 }
1524 if (expected_size_i == 2) {
1525 expected_size_i = trusted_by_source;
1526 }
1527
1528 int node_i =
1529 std::make_tuple(y_node->pmc_id_.is_valid(), x_node->pmc_id_, y_node->file_ids_.size(), main_file_id_i == 1) >
1530 std::make_tuple(x_node->pmc_id_.is_valid(), y_node->pmc_id_, x_node->file_ids_.size(), main_file_id_i == 0);
1531
1532 auto other_node_i = 1 - node_i;
1533 FileNodePtr node = nodes[node_i];
1534 FileNodePtr other_node = nodes[other_node_i];
1535 auto file_view = FileView(node);
1536
1537 LOG(DEBUG) << "Have x_node->pmc_id_ = " << x_node->pmc_id_.get() << ", y_node->pmc_id_ = " << y_node->pmc_id_.get()
1538 << ", x_node_size = " << x_node->file_ids_.size() << ", y_node_size = " << y_node->file_ids_.size()
1539 << ", node_i = " << node_i << ", local_i = " << local_i << ", remote_i = " << remote_i
1540 << ", generate_i = " << generate_i << ", size_i = " << size_i << ", remote_name_i = " << remote_name_i
1541 << ", url_i = " << url_i << ", owner_i = " << owner_i << ", encryption_key_i = " << encryption_key_i
1542 << ", main_file_id_i = " << main_file_id_i << ", trusted_by_source = " << trusted_by_source
1543 << ", x_source = " << x_node->remote_.full_source << ", y_source = " << y_node->remote_.full_source;
1544 if (local_i == other_node_i) {
1545 do_cancel_download(node);
1546 node->set_download_offset(other_node->download_offset_);
1547 node->set_local_location(other_node->local_, other_node->local_ready_size_, other_node->download_offset_,
1548 other_node->local_ready_prefix_size_);
1549 node->download_id_ = other_node->download_id_;
1550 node->download_was_update_file_reference_ = other_node->download_was_update_file_reference_;
1551 node->is_download_started_ |= other_node->is_download_started_;
1552 node->set_download_priority(other_node->download_priority_);
1553 other_node->download_id_ = 0;
1554 other_node->download_was_update_file_reference_ = false;
1555 other_node->is_download_started_ = false;
1556 other_node->download_priority_ = 0;
1557 other_node->download_offset_ = 0;
1558 other_node->local_ready_prefix_size_ = 0;
1559
1560 //do_cancel_generate(node);
1561 //node->set_generate_location(std::move(other_node->generate_));
1562 //node->generate_id_ = other_node->generate_id_;
1563 //node->set_generate_priority(other_node->generate_download_priority_, other_node->generate_upload_priority_);
1564 //other_node->generate_id_ = 0;
1565 //other_node->generate_was_update_ = false;
1566 //other_node->generate_priority_ = 0;
1567 //other_node->generate_download_priority_ = 0;
1568 //other_node->generate_upload_priority_ = 0;
1569 } else {
1570 do_cancel_download(other_node);
1571 //do_cancel_generate(other_node);
1572 }
1573
1574 if (remote_i == other_node_i) {
1575 do_cancel_upload(node);
1576 node->set_new_remote_location(std::move(other_node->remote_));
1577 node->upload_id_ = other_node->upload_id_;
1578 node->upload_was_update_file_reference_ = other_node->upload_was_update_file_reference_;
1579 node->set_upload_priority(other_node->upload_priority_);
1580 node->set_upload_pause(other_node->upload_pause_);
1581 other_node->upload_id_ = 0;
1582 other_node->upload_was_update_file_reference_ = false;
1583 other_node->upload_priority_ = 0;
1584 other_node->set_upload_pause(FileId());
1585 } else {
1586 do_cancel_upload(other_node);
1587 }
1588
1589 if (generate_i == other_node_i) {
1590 do_cancel_generate(node);
1591 node->set_generate_location(std::move(other_node->generate_));
1592 node->generate_id_ = other_node->generate_id_;
1593 node->set_generate_priority(other_node->generate_download_priority_, other_node->generate_upload_priority_);
1594 other_node->generate_id_ = 0;
1595 other_node->generate_priority_ = 0;
1596 other_node->generate_download_priority_ = 0;
1597 other_node->generate_upload_priority_ = 0;
1598 } else {
1599 do_cancel_generate(other_node);
1600 }
1601
1602 if (size_i == other_node_i) {
1603 node->set_size(other_node->size_);
1604 }
1605
1606 if (expected_size_i == other_node_i) {
1607 node->set_expected_size(other_node->expected_size_);
1608 }
1609
1610 if (remote_name_i == other_node_i) {
1611 node->set_remote_name(other_node->remote_name_);
1612 }
1613
1614 if (url_i == other_node_i) {
1615 node->set_url(other_node->url_);
1616 }
1617
1618 if (owner_i == other_node_i) {
1619 node->set_owner_dialog_id(other_node->owner_dialog_id_);
1620 }
1621
1622 if (encryption_key_i == other_node_i) {
1623 node->set_encryption_key(other_node->encryption_key_);
1624 nodes[node_i]->set_encryption_key(nodes[encryption_key_i]->encryption_key_);
1625 }
1626 node->need_load_from_pmc_ |= other_node->need_load_from_pmc_;
1627 node->can_search_locally_ &= other_node->can_search_locally_;
1628 node->upload_prefer_small_ |= other_node->upload_prefer_small_;
1629
1630 if (drop_last_successful_force_reupload_time) {
1631 node->last_successful_force_reupload_time_ = -1e10;
1632 } else if (other_node->last_successful_force_reupload_time_ > node->last_successful_force_reupload_time_) {
1633 node->last_successful_force_reupload_time_ = other_node->last_successful_force_reupload_time_;
1634 }
1635
1636 if (main_file_id_i == other_node_i) {
1637 context_->on_merge_files(other_node->main_file_id_, node->main_file_id_);
1638 node->main_file_id_ = other_node->main_file_id_;
1639 node->main_file_id_priority_ = other_node->main_file_id_priority_;
1640 } else {
1641 context_->on_merge_files(node->main_file_id_, other_node->main_file_id_);
1642 }
1643
1644 bool send_updates_flag = false;
1645 auto other_pmc_id = other_node->pmc_id_;
1646 node->file_ids_.insert(node->file_ids_.end(), other_node->file_ids_.begin(), other_node->file_ids_.end());
1647
1648 for (auto file_id : other_node->file_ids_) {
1649 auto file_id_info = get_file_id_info(file_id);
1650 CHECK(file_id_info->node_id_ == node_ids[other_node_i]);
1651 file_id_info->node_id_ = node_ids[node_i];
1652 send_updates_flag |= file_id_info->send_updates_flag_;
1653 }
1654 other_node = {};
1655
1656 if (send_updates_flag) {
1657 // node might not changed, but other_node might changed, so we need to send update anyway
1658 VLOG(update_file) << "File " << node->main_file_id_ << " has been merged";
1659 node->on_info_changed();
1660 }
1661
1662 // Check if some download/upload queries are ready
1663 for (auto file_id : vector<FileId>(node->file_ids_)) {
1664 auto *info = get_file_id_info(file_id);
1665 if (info->download_priority_ != 0 && file_view.has_local_location()) {
1666 info->download_priority_ = 0;
1667 if (info->download_callback_) {
1668 info->download_callback_->on_download_ok(file_id);
1669 info->download_callback_.reset();
1670 }
1671 }
1672 if (info->upload_priority_ != 0 && file_view.has_active_upload_remote_location()) {
1673 info->upload_priority_ = 0;
1674 if (info->upload_callback_) {
1675 info->upload_callback_->on_upload_ok(file_id, nullptr);
1676 info->upload_callback_.reset();
1677 }
1678 }
1679 }
1680
1681 file_nodes_[node_ids[other_node_i]] = nullptr;
1682
1683 run_generate(node);
1684 run_download(node, false);
1685 run_upload(node, {});
1686
1687 if (other_pmc_id.is_valid()) {
1688 // node might not changed, but we need to merge nodes in pmc anyway
1689 node->on_pmc_changed();
1690 }
1691 try_flush_node_full(node, node_i != remote_i, node_i != local_i, node_i != generate_i, other_pmc_id);
1692
1693 return node->main_file_id_;
1694 }
1695
add_file_source(FileId file_id,FileSourceId file_source_id)1696 void FileManager::add_file_source(FileId file_id, FileSourceId file_source_id) {
1697 auto node = get_file_node(file_id);
1698 if (!node) {
1699 return;
1700 }
1701
1702 CHECK(file_source_id.is_valid());
1703 if (context_->add_file_source(node->main_file_id_, file_source_id)) {
1704 node->on_pmc_changed();
1705 try_flush_node_pmc(node, "add_file_source");
1706 }
1707 }
1708
remove_file_source(FileId file_id,FileSourceId file_source_id)1709 void FileManager::remove_file_source(FileId file_id, FileSourceId file_source_id) {
1710 auto node = get_file_node(file_id);
1711 if (!node) {
1712 return;
1713 }
1714
1715 CHECK(file_source_id.is_valid());
1716 if (context_->remove_file_source(node->main_file_id_, file_source_id)) {
1717 node->on_pmc_changed();
1718 try_flush_node_pmc(node, "remove_file_source");
1719 }
1720 }
1721
change_files_source(FileSourceId file_source_id,const vector<FileId> & old_file_ids,const vector<FileId> & new_file_ids)1722 void FileManager::change_files_source(FileSourceId file_source_id, const vector<FileId> &old_file_ids,
1723 const vector<FileId> &new_file_ids) {
1724 if (old_file_ids == new_file_ids) {
1725 return;
1726 }
1727 CHECK(file_source_id.is_valid());
1728
1729 auto old_main_file_ids = get_main_file_ids(old_file_ids);
1730 auto new_main_file_ids = get_main_file_ids(new_file_ids);
1731 for (auto file_id : old_main_file_ids) {
1732 auto it = new_main_file_ids.find(file_id);
1733 if (it == new_main_file_ids.end()) {
1734 remove_file_source(file_id, file_source_id);
1735 } else {
1736 new_main_file_ids.erase(it);
1737 }
1738 }
1739 for (auto file_id : new_main_file_ids) {
1740 add_file_source(file_id, file_source_id);
1741 }
1742 }
1743
on_file_reference_repaired(FileId file_id,FileSourceId file_source_id,Result<Unit> && result,Promise<Unit> && promise)1744 void FileManager::on_file_reference_repaired(FileId file_id, FileSourceId file_source_id, Result<Unit> &&result,
1745 Promise<Unit> &&promise) {
1746 TRY_STATUS_PROMISE(promise, G()->close_status());
1747
1748 auto file_view = get_file_view(file_id);
1749 CHECK(!file_view.empty());
1750 if (result.is_ok() &&
1751 (!file_view.has_active_upload_remote_location() || !file_view.has_active_download_remote_location())) {
1752 result = Status::Error("No active remote location");
1753 }
1754 if (result.is_error() && result.error().code() != 429 && result.error().code() < 500) {
1755 VLOG(file_references) << "Invalid " << file_source_id << " " << result.error();
1756 remove_file_source(file_id, file_source_id);
1757 }
1758 promise.set_result(std::move(result));
1759 }
1760
get_main_file_ids(const vector<FileId> & file_ids)1761 std::unordered_set<FileId, FileIdHash> FileManager::get_main_file_ids(const vector<FileId> &file_ids) {
1762 std::unordered_set<FileId, FileIdHash> result;
1763 for (auto file_id : file_ids) {
1764 auto node = get_file_node(file_id);
1765 if (node) {
1766 result.insert(node->main_file_id_);
1767 }
1768 }
1769 return result;
1770 }
1771
try_flush_node_full(FileNodePtr node,bool new_remote,bool new_local,bool new_generate,FileDbId other_pmc_id)1772 void FileManager::try_flush_node_full(FileNodePtr node, bool new_remote, bool new_local, bool new_generate,
1773 FileDbId other_pmc_id) {
1774 if (node->need_pmc_flush()) {
1775 if (file_db_) {
1776 load_from_pmc(node, true, true, true);
1777 flush_to_pmc(node, new_remote, new_local, new_generate, "try_flush_node_full");
1778 if (other_pmc_id.is_valid() && node->pmc_id_ != other_pmc_id) {
1779 file_db_->set_file_data_ref(other_pmc_id, node->pmc_id_);
1780 }
1781 }
1782 node->on_pmc_flushed();
1783 }
1784
1785 try_flush_node_info(node, "try_flush_node_full");
1786 }
1787
try_flush_node(FileNodePtr node,const char * source)1788 void FileManager::try_flush_node(FileNodePtr node, const char *source) {
1789 try_flush_node_pmc(node, source);
1790 try_flush_node_info(node, source);
1791 }
1792
try_flush_node_pmc(FileNodePtr node,const char * source)1793 void FileManager::try_flush_node_pmc(FileNodePtr node, const char *source) {
1794 if (node->need_pmc_flush()) {
1795 if (file_db_) {
1796 load_from_pmc(node, true, true, true);
1797 flush_to_pmc(node, false, false, false, source);
1798 }
1799 node->on_pmc_flushed();
1800 }
1801 }
1802
try_flush_node_info(FileNodePtr node,const char * source)1803 void FileManager::try_flush_node_info(FileNodePtr node, const char *source) {
1804 if (node->need_info_flush()) {
1805 for (auto file_id : vector<FileId>(node->file_ids_)) {
1806 auto *info = get_file_id_info(file_id);
1807 if (info->send_updates_flag_) {
1808 VLOG(update_file) << "Send UpdateFile about file " << file_id << " from " << source;
1809 context_->on_file_updated(file_id);
1810 }
1811 }
1812 node->on_info_flushed();
1813 }
1814 }
1815
clear_from_pmc(FileNodePtr node)1816 void FileManager::clear_from_pmc(FileNodePtr node) {
1817 if (!file_db_) {
1818 return;
1819 }
1820 if (node->pmc_id_.empty()) {
1821 return;
1822 }
1823
1824 LOG(INFO) << "Delete files " << format::as_array(node->file_ids_) << " from pmc";
1825 FileData data;
1826 auto file_view = FileView(node);
1827 if (file_view.has_local_location()) {
1828 data.local_ = node->local_;
1829 }
1830 if (file_view.has_remote_location()) {
1831 data.remote_ = RemoteFileLocation(*node->remote_.full);
1832 }
1833 if (file_view.has_generate_location()) {
1834 data.generate_ = make_unique<FullGenerateFileLocation>(*node->generate_);
1835 }
1836 file_db_->clear_file_data(node->pmc_id_, data);
1837 node->pmc_id_ = FileDbId();
1838 }
1839
flush_to_pmc(FileNodePtr node,bool new_remote,bool new_local,bool new_generate,const char * source)1840 void FileManager::flush_to_pmc(FileNodePtr node, bool new_remote, bool new_local, bool new_generate,
1841 const char *source) {
1842 if (!file_db_) {
1843 return;
1844 }
1845 FileView view(node);
1846 bool create_flag = false;
1847 if (node->pmc_id_.empty()) {
1848 create_flag = true;
1849 node->pmc_id_ = file_db_->create_pmc_id();
1850 }
1851
1852 FileData data;
1853 data.pmc_id_ = node->pmc_id_.get();
1854 data.local_ = node->local_;
1855 if (data.local_.type() == LocalFileLocation::Type::Full) {
1856 prepare_path_for_pmc(data.local_.full().file_type_, data.local_.full().path_);
1857 }
1858 if (node->remote_.full) {
1859 data.remote_ = RemoteFileLocation(node->remote_.full.value());
1860 } else if (node->remote_.partial) {
1861 data.remote_ = RemoteFileLocation(*node->remote_.partial);
1862 }
1863 if (node->generate_ != nullptr && !begins_with(node->generate_->conversion_, "#file_id#")) {
1864 data.generate_ = make_unique<FullGenerateFileLocation>(*node->generate_);
1865 }
1866
1867 // TODO: not needed when GenerateLocation has constant conversion
1868 if (data.remote_.type() != RemoteFileLocation::Type::Full && data.local_.type() != LocalFileLocation::Type::Full) {
1869 data.local_ = LocalFileLocation();
1870 data.remote_ = RemoteFileLocation();
1871 }
1872 if (data.remote_.type() != RemoteFileLocation::Type::Full && node->encryption_key_.is_secure()) {
1873 data.remote_ = RemoteFileLocation();
1874 }
1875
1876 data.size_ = node->size_;
1877 data.expected_size_ = node->expected_size_;
1878 data.remote_name_ = node->remote_name_;
1879 data.encryption_key_ = node->encryption_key_;
1880 data.url_ = node->url_;
1881 data.owner_dialog_id_ = node->owner_dialog_id_;
1882 data.file_source_ids_ = context_->get_some_file_sources(view.file_id());
1883 VLOG(file_references) << "Save file " << view.file_id() << " to database with " << data.file_source_ids_ << " from "
1884 << source;
1885
1886 file_db_->set_file_data(node->pmc_id_, data, (create_flag || new_remote), (create_flag || new_local),
1887 (create_flag || new_generate));
1888 }
1889
get_file_node_raw(FileId file_id,FileNodeId * file_node_id)1890 FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_id) {
1891 if (file_id.get() <= 0 || file_id.get() >= static_cast<int32>(file_id_info_.size())) {
1892 return nullptr;
1893 }
1894 FileNodeId node_id = file_id_info_[file_id.get()].node_id_;
1895 if (node_id == 0) {
1896 return nullptr;
1897 }
1898 if (file_node_id != nullptr) {
1899 *file_node_id = node_id;
1900 }
1901 return file_nodes_[node_id].get();
1902 }
1903
get_sync_file_node(FileId file_id)1904 FileNodePtr FileManager::get_sync_file_node(FileId file_id) {
1905 auto file_node = get_file_node(file_id);
1906 if (!file_node) {
1907 return {};
1908 }
1909 load_from_pmc(file_node, true, true, true);
1910 return file_node;
1911 }
1912
load_from_pmc(FileNodePtr node,bool new_remote,bool new_local,bool new_generate)1913 void FileManager::load_from_pmc(FileNodePtr node, bool new_remote, bool new_local, bool new_generate) {
1914 if (!node->need_load_from_pmc_) {
1915 return;
1916 }
1917 auto file_id = node->main_file_id_;
1918 node->need_load_from_pmc_ = false;
1919 if (!file_db_) {
1920 return;
1921 }
1922 auto file_view = get_file_view(file_id);
1923 CHECK(!file_view.empty());
1924
1925 FullRemoteFileLocation remote;
1926 FullLocalFileLocation local;
1927 FullGenerateFileLocation generate;
1928 new_remote &= file_view.has_remote_location();
1929 if (new_remote) {
1930 remote = file_view.remote_location();
1931 }
1932 new_local &= file_view.has_local_location();
1933 if (new_local) {
1934 local = file_view.local_location();
1935 prepare_path_for_pmc(local.file_type_, local.path_);
1936 }
1937 new_generate &= file_view.has_generate_location();
1938 if (new_generate) {
1939 generate = file_view.generate_location();
1940 }
1941
1942 LOG(DEBUG) << "Load from pmc " << file_id << "/" << file_view.file_id() << ", new_remote = " << new_remote
1943 << ", new_local = " << new_local << ", new_generate = " << new_generate;
1944 auto load = [&](auto location) {
1945 TRY_RESULT(file_data, file_db_->get_file_data_sync(location));
1946 TRY_RESULT(new_file_id,
1947 register_file(std::move(file_data), FileLocationSource::FromDatabase, "load_from_pmc", false));
1948 TRY_RESULT(main_file_id, merge(file_id, new_file_id));
1949 file_id = main_file_id;
1950 return Status::OK();
1951 };
1952 if (new_remote) {
1953 load(remote).ignore();
1954 }
1955 if (new_local) {
1956 load(local).ignore();
1957 }
1958 if (new_generate) {
1959 load(generate).ignore();
1960 }
1961 }
1962
set_encryption_key(FileId file_id,FileEncryptionKey key)1963 bool FileManager::set_encryption_key(FileId file_id, FileEncryptionKey key) {
1964 auto node = get_sync_file_node(file_id);
1965 if (!node) {
1966 return false;
1967 }
1968 auto view = FileView(node);
1969 if (view.has_local_location() && view.has_remote_location()) {
1970 return false;
1971 }
1972 if (!node->encryption_key_.empty()) {
1973 return false;
1974 }
1975 node->set_encryption_key(std::move(key));
1976 try_flush_node_pmc(node, "set_encryption_key");
1977 return true;
1978 }
1979
set_content(FileId file_id,BufferSlice bytes)1980 bool FileManager::set_content(FileId file_id, BufferSlice bytes) {
1981 if (G()->shared_config().get_option_boolean("ignore_inline_thumbnails")) {
1982 return false;
1983 }
1984
1985 auto node = get_sync_file_node(file_id);
1986 if (!node) {
1987 return false;
1988 }
1989
1990 if (node->local_.type() == LocalFileLocation::Type::Full) {
1991 // There was no download so we don't need an update
1992 return true;
1993 }
1994
1995 if (node->download_priority_ == FROM_BYTES_PRIORITY) {
1996 return true;
1997 }
1998
1999 do_cancel_download(node);
2000
2001 auto *file_info = get_file_id_info(file_id);
2002 file_info->download_priority_ = FROM_BYTES_PRIORITY;
2003
2004 node->set_download_priority(FROM_BYTES_PRIORITY);
2005
2006 QueryId id = queries_container_.create(Query{file_id, Query::Type::SetContent});
2007 node->download_id_ = id;
2008 node->is_download_started_ = true;
2009 send_closure(file_load_manager_, &FileLoadManager::from_bytes, id, node->remote_.full.value().file_type_,
2010 std::move(bytes), node->suggested_path());
2011 return true;
2012 }
2013
get_content(FileId file_id,Promise<BufferSlice> promise)2014 void FileManager::get_content(FileId file_id, Promise<BufferSlice> promise) {
2015 auto node = get_sync_file_node(file_id);
2016 if (!node) {
2017 return promise.set_error(Status::Error("Unknown file_id"));
2018 }
2019 auto status = check_local_location(node);
2020 status.ignore();
2021
2022 auto file_view = FileView(node);
2023 if (!file_view.has_local_location()) {
2024 return promise.set_error(Status::Error("No local location"));
2025 }
2026
2027 send_closure(file_load_manager_, &FileLoadManager::get_content, node->local_.full(), std::move(promise));
2028 }
2029
read_file_part(FileId file_id,int32 offset,int32 count,int left_tries,Promise<td_api::object_ptr<td_api::filePart>> promise)2030 void FileManager::read_file_part(FileId file_id, int32 offset, int32 count, int left_tries,
2031 Promise<td_api::object_ptr<td_api::filePart>> promise) {
2032 TRY_STATUS_PROMISE(promise, G()->close_status());
2033
2034 if (!file_id.is_valid()) {
2035 return promise.set_error(Status::Error(400, "File identifier is invalid"));
2036 }
2037 auto node = get_sync_file_node(file_id);
2038 if (!node) {
2039 return promise.set_error(Status::Error(400, "File not found"));
2040 }
2041 if (offset < 0) {
2042 return promise.set_error(Status::Error(400, "Parameter offset must be non-negative"));
2043 }
2044 if (count < 0) {
2045 return promise.set_error(Status::Error(400, "Parameter count must be non-negative"));
2046 }
2047
2048 auto file_view = FileView(node);
2049
2050 if (count == 0) {
2051 count = narrow_cast<int32>(file_view.downloaded_prefix(offset));
2052 if (count == 0) {
2053 return promise.set_value(td_api::make_object<td_api::filePart>());
2054 }
2055 } else if (file_view.downloaded_prefix(offset) < static_cast<int64>(count)) {
2056 // TODO this check is safer to do in another thread
2057 return promise.set_error(Status::Error(400, "There is not enough downloaded bytes in the file to read"));
2058 }
2059
2060 const string *path = nullptr;
2061 bool is_partial = false;
2062 if (file_view.has_local_location()) {
2063 path = &file_view.local_location().path_;
2064 if (!begins_with(*path, get_files_dir(file_view.get_type()))) {
2065 return promise.set_error(Status::Error(400, "File is not inside the cache"));
2066 }
2067 } else {
2068 CHECK(node->local_.type() == LocalFileLocation::Type::Partial);
2069 path = &node->local_.partial().path_;
2070 is_partial = true;
2071 }
2072
2073 // TODO move file reading to another thread
2074 auto r_bytes = [&]() -> Result<string> {
2075 TRY_RESULT(fd, FileFd::open(*path, FileFd::Read));
2076 string data;
2077 data.resize(count);
2078 TRY_RESULT(read_bytes, fd.pread(data, offset));
2079 if (read_bytes != static_cast<size_t>(count)) {
2080 return Status::Error("Read less bytes than expected");
2081 }
2082 return std::move(data);
2083 }();
2084 if (r_bytes.is_error()) {
2085 LOG(INFO) << "Failed to read file bytes: " << r_bytes.error();
2086 if (--left_tries == 0 || !is_partial) {
2087 return promise.set_error(Status::Error(400, "Failed to read the file"));
2088 }
2089
2090 // the temporary file could be moved from temp to persistent folder
2091 // we need to wait for the corresponding update and repeat the reading
2092 create_actor<SleepActor>("RepeatReadFilePartActor", 0.01,
2093 PromiseCreator::lambda([actor_id = actor_id(this), file_id, offset, count, left_tries,
2094 promise = std::move(promise)](Result<Unit> result) mutable {
2095 send_closure(actor_id, &FileManager::read_file_part, file_id, offset, count, left_tries,
2096 std::move(promise));
2097 }))
2098 .release();
2099 return;
2100 }
2101
2102 auto result = td_api::make_object<td_api::filePart>();
2103 result->data_ = r_bytes.move_as_ok();
2104 promise.set_value(std::move(result));
2105 }
2106
delete_file(FileId file_id,Promise<Unit> promise,const char * source)2107 void FileManager::delete_file(FileId file_id, Promise<Unit> promise, const char *source) {
2108 LOG(INFO) << "Trying to delete file " << file_id << " from " << source;
2109 auto node = get_sync_file_node(file_id);
2110 if (!node) {
2111 return promise.set_value(Unit());
2112 }
2113
2114 auto file_view = FileView(node);
2115
2116 // TODO review delete condition
2117 if (file_view.has_local_location()) {
2118 if (begins_with(file_view.local_location().path_, get_files_dir(file_view.get_type()))) {
2119 LOG(INFO) << "Unlink file " << file_id << " at " << file_view.local_location().path_;
2120 clear_from_pmc(node);
2121
2122 context_->on_new_file(-file_view.size(), -file_view.get_allocated_local_size(), -1);
2123 unlink(file_view.local_location().path_).ignore();
2124 node->drop_local_location();
2125 try_flush_node(node, "delete_file 1");
2126 }
2127 } else {
2128 if (file_view.get_type() == FileType::Encrypted) {
2129 clear_from_pmc(node);
2130 }
2131 if (node->local_.type() == LocalFileLocation::Type::Partial) {
2132 LOG(INFO) << "Unlink partial file " << file_id << " at " << node->local_.partial().path_;
2133 unlink(node->local_.partial().path_).ignore();
2134 node->drop_local_location();
2135 try_flush_node(node, "delete_file 2");
2136 }
2137 }
2138
2139 promise.set_value(Unit());
2140 }
2141
download(FileId file_id,std::shared_ptr<DownloadCallback> callback,int32 new_priority,int64 offset,int64 limit)2142 void FileManager::download(FileId file_id, std::shared_ptr<DownloadCallback> callback, int32 new_priority, int64 offset,
2143 int64 limit) {
2144 LOG(INFO) << "Download file " << file_id << " with priority " << new_priority;
2145 auto node = get_sync_file_node(file_id);
2146 if (!node) {
2147 LOG(INFO) << "File " << file_id << " not found";
2148 if (callback) {
2149 callback->on_download_error(file_id, Status::Error("File not found"));
2150 }
2151 return;
2152 }
2153
2154 if (node->local_.type() == LocalFileLocation::Type::Full) {
2155 auto status = check_local_location(node);
2156 if (status.is_error()) {
2157 LOG(WARNING) << "Need to redownload file " << file_id << ": " << status.error();
2158 } else {
2159 LOG(INFO) << "File " << file_id << " is already downloaded";
2160 if (callback) {
2161 callback->on_download_ok(file_id);
2162 }
2163 return;
2164 }
2165 } else if (node->local_.type() == LocalFileLocation::Type::Partial) {
2166 auto status = check_local_location(node);
2167 if (status.is_error()) {
2168 LOG(WARNING) << "Need to download file " << file_id << " from beginning: " << status.error();
2169 }
2170 }
2171
2172 FileView file_view(node);
2173 if (!file_view.can_download_from_server() && !file_view.can_generate()) {
2174 LOG(INFO) << "File " << file_id << " can't be downloaded";
2175 if (callback) {
2176 callback->on_download_error(file_id, Status::Error("Can't download or generate file"));
2177 }
2178 return;
2179 }
2180
2181 if (new_priority == -1) {
2182 if (node->is_download_started_) {
2183 LOG(INFO) << "File " << file_id << " is being downloaded";
2184 return;
2185 }
2186 new_priority = 0;
2187 }
2188
2189 LOG(INFO) << "Change download priority of file " << file_id << " to " << new_priority;
2190 node->set_download_offset(offset);
2191 node->set_download_limit(limit);
2192 auto *file_info = get_file_id_info(file_id);
2193 CHECK(new_priority == 0 || callback);
2194 if (file_info->download_callback_ != nullptr && file_info->download_callback_.get() != callback.get()) {
2195 // the callback will be destroyed soon and lost forever
2196 // this would be an error and should never happen, unless we cancel previous download query
2197 // in that case we send an error to the callback
2198 CHECK(new_priority == 0);
2199 file_info->download_callback_->on_download_error(file_id, Status::Error(200, "Canceled"));
2200 }
2201 file_info->download_priority_ = narrow_cast<int8>(new_priority);
2202 file_info->download_callback_ = std::move(callback);
2203 // TODO: send current progress?
2204
2205 run_generate(node);
2206 run_download(node, true);
2207
2208 try_flush_node(node, "download");
2209 }
2210
run_download(FileNodePtr node,bool force_update_priority)2211 void FileManager::run_download(FileNodePtr node, bool force_update_priority) {
2212 int8 priority = 0;
2213 for (auto id : node->file_ids_) {
2214 auto *info = get_file_id_info(id);
2215 if (info->download_priority_ > priority) {
2216 priority = info->download_priority_;
2217 }
2218 }
2219
2220 auto old_priority = node->download_priority_;
2221
2222 if (priority == 0) {
2223 node->set_download_priority(priority);
2224 if (old_priority != 0) {
2225 LOG(INFO) << "Cancel downloading of file " << node->main_file_id_;
2226 do_cancel_download(node);
2227 }
2228 return;
2229 }
2230
2231 if (node->need_load_from_pmc_) {
2232 LOG(INFO) << "Skip run_download, because file " << node->main_file_id_ << " needs to be loaded from PMC";
2233 return;
2234 }
2235 if (node->generate_id_) {
2236 LOG(INFO) << "Skip run_download, because file " << node->main_file_id_ << " is being generated";
2237 return;
2238 }
2239 auto file_view = FileView(node);
2240 if (!file_view.can_download_from_server()) {
2241 LOG(INFO) << "Skip run_download, because file " << node->main_file_id_ << " can't be downloaded from server";
2242 return;
2243 }
2244 node->set_download_priority(priority);
2245 bool need_update_offset = node->is_download_offset_dirty_;
2246 node->is_download_offset_dirty_ = false;
2247
2248 bool need_update_limit = node->is_download_limit_dirty_;
2249 node->is_download_limit_dirty_ = false;
2250
2251 if (old_priority != 0) {
2252 LOG(INFO) << "Update download offset and limits of file " << node->main_file_id_;
2253 CHECK(node->download_id_ != 0);
2254 if (force_update_priority || priority != old_priority) {
2255 send_closure(file_load_manager_, &FileLoadManager::update_priority, node->download_id_, priority);
2256 }
2257 if (need_update_limit || need_update_offset) {
2258 auto download_offset = node->download_offset_;
2259 auto download_limit = node->download_limit_;
2260 if (file_view.is_encrypted_any()) {
2261 CHECK(download_offset <= MAX_FILE_SIZE);
2262 CHECK(download_limit <= std::numeric_limits<int32>::max());
2263 download_limit += download_offset;
2264 download_offset = 0;
2265 }
2266 send_closure(file_load_manager_, &FileLoadManager::update_downloaded_part, node->download_id_, download_offset,
2267 download_limit);
2268 }
2269 return;
2270 }
2271
2272 CHECK(node->download_id_ == 0);
2273 CHECK(!node->file_ids_.empty());
2274 auto file_id = node->main_file_id_;
2275
2276 if (node->need_reload_photo_ && file_view.may_reload_photo()) {
2277 LOG(INFO) << "Reload photo from file " << node->main_file_id_;
2278 QueryId id = queries_container_.create(Query{file_id, Query::Type::DownloadReloadDialog});
2279 node->download_id_ = id;
2280 context_->reload_photo(file_view.remote_location().get_source(),
2281 PromiseCreator::lambda([id, actor_id = actor_id(this), file_id](Result<Unit> res) {
2282 Status error;
2283 if (res.is_ok()) {
2284 error = Status::Error("FILE_DOWNLOAD_ID_INVALID");
2285 } else {
2286 error = res.move_as_error();
2287 }
2288 VLOG(file_references)
2289 << "Got result from reload photo for file " << file_id << ": " << error;
2290 send_closure(actor_id, &FileManager::on_error, id, std::move(error));
2291 }));
2292 node->need_reload_photo_ = false;
2293 return;
2294 }
2295
2296 // If file reference is needed
2297 if (!file_view.has_active_download_remote_location()) {
2298 VLOG(file_references) << "Do not have valid file_reference for file " << file_id;
2299 QueryId id = queries_container_.create(Query{file_id, Query::Type::DownloadWaitFileReference});
2300 node->download_id_ = id;
2301 if (node->download_was_update_file_reference_) {
2302 on_error(id, Status::Error("Can't download file: have no valid file reference"));
2303 return;
2304 }
2305 node->download_was_update_file_reference_ = true;
2306
2307 context_->repair_file_reference(
2308 file_id, PromiseCreator::lambda([id, actor_id = actor_id(this), file_id](Result<Unit> res) {
2309 Status error;
2310 if (res.is_ok()) {
2311 error = Status::Error("FILE_DOWNLOAD_RESTART_WITH_FILE_REFERENCE");
2312 } else {
2313 error = res.move_as_error();
2314 }
2315 VLOG(file_references) << "Got result from FileSourceManager for file " << file_id << ": " << error;
2316 send_closure(actor_id, &FileManager::on_error, id, std::move(error));
2317 }));
2318 return;
2319 }
2320
2321 QueryId id = queries_container_.create(Query{file_id, Query::Type::Download});
2322 node->download_id_ = id;
2323 node->is_download_started_ = false;
2324 LOG(INFO) << "Run download of file " << file_id << " of size " << node->size_ << " from "
2325 << node->remote_.full.value() << " with suggested name " << node->suggested_path() << " and encyption key "
2326 << node->encryption_key_;
2327 auto download_offset = node->download_offset_;
2328 auto download_limit = node->download_limit_;
2329 if (file_view.is_encrypted_any()) {
2330 CHECK(download_offset <= MAX_FILE_SIZE);
2331 CHECK(download_limit <= std::numeric_limits<int32>::max());
2332 download_limit += download_offset;
2333 download_offset = 0;
2334 }
2335 send_closure(file_load_manager_, &FileLoadManager::download, id, node->remote_.full.value(), node->local_,
2336 node->size_, node->suggested_path(), node->encryption_key_, node->can_search_locally_, download_offset,
2337 download_limit, priority);
2338 }
2339
2340 class FileManager::ForceUploadActor final : public Actor {
2341 public:
ForceUploadActor(FileManager * file_manager,FileId file_id,std::shared_ptr<FileManager::UploadCallback> callback,int32 new_priority,uint64 upload_order,bool prefer_small,ActorShared<> parent)2342 ForceUploadActor(FileManager *file_manager, FileId file_id, std::shared_ptr<FileManager::UploadCallback> callback,
2343 int32 new_priority, uint64 upload_order, bool prefer_small, ActorShared<> parent)
2344 : file_manager_(file_manager)
2345 , file_id_(file_id)
2346 , callback_(std::move(callback))
2347 , new_priority_(new_priority)
2348 , upload_order_(upload_order)
2349 , prefer_small_(prefer_small)
2350 , parent_(std::move(parent)) {
2351 }
2352
2353 private:
2354 FileManager *file_manager_;
2355 FileId file_id_;
2356 std::shared_ptr<FileManager::UploadCallback> callback_;
2357 int32 new_priority_;
2358 uint64 upload_order_;
2359 bool prefer_small_;
2360 ActorShared<> parent_;
2361 bool is_active_{false};
2362 int attempt_{0};
2363
2364 class UploadCallback final : public FileManager::UploadCallback {
2365 public:
UploadCallback(ActorId<ForceUploadActor> callback)2366 explicit UploadCallback(ActorId<ForceUploadActor> callback) : callback_(std::move(callback)) {
2367 }
on_upload_ok(FileId file_id,tl_object_ptr<telegram_api::InputFile> input_file)2368 void on_upload_ok(FileId file_id, tl_object_ptr<telegram_api::InputFile> input_file) final {
2369 send_closure(std::move(callback_), &ForceUploadActor::on_upload_ok, std::move(input_file));
2370 }
2371
on_upload_encrypted_ok(FileId file_id,tl_object_ptr<telegram_api::InputEncryptedFile> input_file)2372 void on_upload_encrypted_ok(FileId file_id, tl_object_ptr<telegram_api::InputEncryptedFile> input_file) final {
2373 send_closure(std::move(callback_), &ForceUploadActor::on_upload_encrypted_ok, std::move(input_file));
2374 }
2375
on_upload_secure_ok(FileId file_id,tl_object_ptr<telegram_api::InputSecureFile> input_file)2376 void on_upload_secure_ok(FileId file_id, tl_object_ptr<telegram_api::InputSecureFile> input_file) final {
2377 send_closure(std::move(callback_), &ForceUploadActor::on_upload_secure_ok, std::move(input_file));
2378 }
2379
on_upload_error(FileId file_id,Status error)2380 void on_upload_error(FileId file_id, Status error) final {
2381 send_closure(std::move(callback_), &ForceUploadActor::on_upload_error, std::move(error));
2382 }
~UploadCallback()2383 ~UploadCallback() final {
2384 if (callback_.empty()) {
2385 return;
2386 }
2387 send_closure(std::move(callback_), &ForceUploadActor::on_upload_error, Status::Error("Canceled"));
2388 }
2389
2390 private:
2391 ActorId<ForceUploadActor> callback_;
2392 };
2393
on_upload_ok(tl_object_ptr<telegram_api::InputFile> input_file)2394 void on_upload_ok(tl_object_ptr<telegram_api::InputFile> input_file) {
2395 is_active_ = false;
2396 if (input_file || is_ready()) {
2397 callback_->on_upload_ok(file_id_, std::move(input_file));
2398 on_ok();
2399 } else {
2400 loop();
2401 }
2402 }
2403
on_upload_encrypted_ok(tl_object_ptr<telegram_api::InputEncryptedFile> input_file)2404 void on_upload_encrypted_ok(tl_object_ptr<telegram_api::InputEncryptedFile> input_file) {
2405 is_active_ = false;
2406 if (input_file || is_ready()) {
2407 callback_->on_upload_encrypted_ok(file_id_, std::move(input_file));
2408 on_ok();
2409 } else {
2410 loop();
2411 }
2412 }
2413
on_upload_secure_ok(tl_object_ptr<telegram_api::InputSecureFile> input_file)2414 void on_upload_secure_ok(tl_object_ptr<telegram_api::InputSecureFile> input_file) {
2415 is_active_ = false;
2416 if (input_file || is_ready()) {
2417 callback_->on_upload_secure_ok(file_id_, std::move(input_file));
2418 on_ok();
2419 } else {
2420 loop();
2421 }
2422 }
2423
is_ready() const2424 bool is_ready() const {
2425 return !G()->close_flag() && file_manager_->get_file_view(file_id_).has_active_upload_remote_location();
2426 }
2427
on_ok()2428 void on_ok() {
2429 callback_.reset();
2430 send_closure(G()->file_manager(), &FileManager::on_force_reupload_success, file_id_);
2431 stop();
2432 }
2433
on_upload_error(Status error)2434 void on_upload_error(Status error) {
2435 if (attempt_ == 2) {
2436 callback_->on_upload_error(file_id_, std::move(error));
2437 callback_.reset();
2438 stop();
2439 } else {
2440 is_active_ = false;
2441 loop();
2442 }
2443 }
2444
create_callback()2445 auto create_callback() {
2446 return std::make_shared<UploadCallback>(actor_id(this));
2447 }
2448
loop()2449 void loop() final {
2450 if (is_active_) {
2451 return;
2452 }
2453 if (G()->close_flag()) {
2454 return stop();
2455 }
2456
2457 is_active_ = true;
2458 attempt_++;
2459 send_closure(G()->file_manager(), &FileManager::resume_upload, file_id_, std::vector<int>(), create_callback(),
2460 new_priority_, upload_order_, attempt_ == 2, prefer_small_);
2461 }
2462
tear_down()2463 void tear_down() final {
2464 if (callback_) {
2465 callback_->on_upload_error(file_id_, Status::Error("Canceled"));
2466 }
2467 }
2468 };
2469
on_force_reupload_success(FileId file_id)2470 void FileManager::on_force_reupload_success(FileId file_id) {
2471 auto node = get_sync_file_node(file_id);
2472 CHECK(node);
2473 if (!node->remote_.is_full_alive) { // do not update for multiple simultaneous uploads
2474 node->last_successful_force_reupload_time_ = Time::now();
2475 }
2476 }
2477
resume_upload(FileId file_id,std::vector<int> bad_parts,std::shared_ptr<UploadCallback> callback,int32 new_priority,uint64 upload_order,bool force,bool prefer_small)2478 void FileManager::resume_upload(FileId file_id, std::vector<int> bad_parts, std::shared_ptr<UploadCallback> callback,
2479 int32 new_priority, uint64 upload_order, bool force, bool prefer_small) {
2480 auto node = get_sync_file_node(file_id);
2481 if (!node) {
2482 LOG(INFO) << "File " << file_id << " not found";
2483 if (callback) {
2484 callback->on_upload_error(file_id, Status::Error("File not found"));
2485 }
2486 return;
2487 }
2488
2489 if (bad_parts.size() == 1 && bad_parts[0] == -1) {
2490 if (node->last_successful_force_reupload_time_ >= Time::now() - 60) {
2491 LOG(INFO) << "Recently reuploaded file " << file_id << ", do not try again";
2492 if (callback) {
2493 callback->on_upload_error(file_id, Status::Error("Failed to reupload file"));
2494 }
2495 return;
2496 }
2497
2498 create_actor<ForceUploadActor>("ForceUploadActor", this, file_id, std::move(callback), new_priority, upload_order,
2499 prefer_small, context_->create_reference())
2500 .release();
2501 return;
2502 }
2503 LOG(INFO) << "Resume upload of file " << file_id << " with priority " << new_priority << " and force = " << force;
2504
2505 if (force) {
2506 node->remote_.is_full_alive = false;
2507 }
2508 if (prefer_small) {
2509 node->upload_prefer_small_ = true;
2510 }
2511 if (node->upload_pause_ == file_id) {
2512 node->set_upload_pause(FileId());
2513 }
2514 FileView file_view(node);
2515 if (file_view.has_active_upload_remote_location() && file_view.get_type() != FileType::Thumbnail &&
2516 file_view.get_type() != FileType::EncryptedThumbnail && file_view.get_type() != FileType::Background) {
2517 LOG(INFO) << "File " << file_id << " is already uploaded";
2518 if (callback) {
2519 callback->on_upload_ok(file_id, nullptr);
2520 }
2521 return;
2522 }
2523
2524 if (file_view.has_local_location()) {
2525 auto status = check_local_location(node);
2526 if (status.is_error()) {
2527 LOG(INFO) << "Full local location of file " << file_id << " for upload is invalid: " << status;
2528 }
2529 }
2530
2531 if (!file_view.has_local_location() && !file_view.has_generate_location() && !file_view.has_alive_remote_location()) {
2532 LOG(INFO) << "File " << file_id << " can't be uploaded";
2533 if (callback) {
2534 callback->on_upload_error(file_id,
2535 Status::Error("Need full local (or generate, or inactive remote) location for upload"));
2536 }
2537 return;
2538 }
2539 if (file_view.get_type() == FileType::Thumbnail &&
2540 (!file_view.has_local_location() && file_view.can_download_from_server())) {
2541 // TODO
2542 if (callback) {
2543 callback->on_upload_error(file_id, Status::Error("Failed to upload thumbnail without local location"));
2544 }
2545 return;
2546 }
2547
2548 LOG(INFO) << "Change upload priority of file " << file_id << " to " << new_priority;
2549 auto *file_info = get_file_id_info(file_id);
2550 CHECK(new_priority == 0 || callback);
2551 file_info->upload_order_ = upload_order;
2552 file_info->upload_priority_ = narrow_cast<int8>(new_priority);
2553 file_info->upload_callback_ = std::move(callback);
2554 // TODO: send current progress?
2555
2556 run_generate(node);
2557 run_upload(node, std::move(bad_parts));
2558 try_flush_node(node, "resume_upload");
2559 }
2560
delete_partial_remote_location(FileId file_id)2561 bool FileManager::delete_partial_remote_location(FileId file_id) {
2562 auto node = get_sync_file_node(file_id);
2563 if (!node) {
2564 LOG(INFO) << "Wrong file identifier " << file_id;
2565 return false;
2566 }
2567 if (node->upload_pause_ == file_id) {
2568 node->set_upload_pause(FileId());
2569 }
2570 if (node->remote_.is_full_alive) {
2571 LOG(INFO) << "File " << file_id << " is already uploaded";
2572 return true;
2573 }
2574
2575 node->delete_partial_remote_location();
2576 auto *file_info = get_file_id_info(file_id);
2577 file_info->upload_priority_ = 0;
2578
2579 if (node->local_.type() != LocalFileLocation::Type::Full) {
2580 LOG(INFO) << "Need full local location to upload file " << file_id;
2581 return false;
2582 }
2583
2584 auto status = check_local_location(node);
2585 if (status.is_error()) {
2586 LOG(INFO) << "Need full local location to upload file " << file_id << ": " << status;
2587 return false;
2588 }
2589
2590 run_upload(node, std::vector<int>());
2591 try_flush_node(node, "delete_partial_remote_location");
2592 return true;
2593 }
2594
delete_file_reference(FileId file_id,Slice file_reference)2595 void FileManager::delete_file_reference(FileId file_id, Slice file_reference) {
2596 VLOG(file_references) << "Delete file reference of file " << file_id << " "
2597 << tag("reference_base64", base64_encode(file_reference));
2598 auto node = get_sync_file_node(file_id);
2599 if (!node) {
2600 LOG(ERROR) << "Wrong file identifier " << file_id;
2601 return;
2602 }
2603 node->delete_file_reference(file_reference);
2604 auto remote = get_remote(file_id.get_remote());
2605 if (remote != nullptr) {
2606 VLOG(file_references) << "Do delete file reference of remote file " << file_id;
2607 if (remote->delete_file_reference(file_reference)) {
2608 VLOG(file_references) << "Successfully deleted file reference of remote file " << file_id;
2609 node->upload_was_update_file_reference_ = false;
2610 node->download_was_update_file_reference_ = false;
2611 node->on_pmc_changed();
2612 }
2613 }
2614 try_flush_node_pmc(node, "delete_file_reference");
2615 }
2616
external_file_generate_write_part(int64 id,int32 offset,string data,Promise<> promise)2617 void FileManager::external_file_generate_write_part(int64 id, int32 offset, string data, Promise<> promise) {
2618 send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_write_part, id, offset,
2619 std::move(data), std::move(promise));
2620 }
2621
external_file_generate_progress(int64 id,int32 expected_size,int32 local_prefix_size,Promise<> promise)2622 void FileManager::external_file_generate_progress(int64 id, int32 expected_size, int32 local_prefix_size,
2623 Promise<> promise) {
2624 send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_progress, id, expected_size,
2625 local_prefix_size, std::move(promise));
2626 }
2627
external_file_generate_finish(int64 id,Status status,Promise<> promise)2628 void FileManager::external_file_generate_finish(int64 id, Status status, Promise<> promise) {
2629 send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_finish, id, std::move(status),
2630 std::move(promise));
2631 }
2632
run_generate(FileNodePtr node)2633 void FileManager::run_generate(FileNodePtr node) {
2634 if (node->need_load_from_pmc_) {
2635 LOG(INFO) << "Skip run_generate, because file " << node->main_file_id_ << " needs to be loaded from PMC";
2636 return;
2637 }
2638 FileView file_view(node);
2639 if (!file_view.can_generate()) {
2640 // LOG(INFO) << "Skip run_generate, because file " << node->main_file_id_ << " can't be generated";
2641 return;
2642 }
2643 if (file_view.has_local_location()) {
2644 LOG(INFO) << "Skip run_generate, because file " << node->main_file_id_ << " has local location";
2645 return;
2646 }
2647 if (file_view.can_download_from_server()) {
2648 LOG(INFO) << "Skip run_generate, because file " << node->main_file_id_ << " can be downloaded from server";
2649 return;
2650 }
2651
2652 int8 download_priority = 0;
2653 int8 upload_priority = 0;
2654 FileId file_id = node->main_file_id_;
2655 for (auto id : node->file_ids_) {
2656 auto *info = get_file_id_info(id);
2657 if (info->download_priority_ > download_priority) {
2658 download_priority = info->download_priority_;
2659 if (download_priority > upload_priority) {
2660 file_id = id;
2661 }
2662 }
2663 if (info->upload_priority_ > upload_priority) {
2664 upload_priority = info->upload_priority_;
2665 if (upload_priority > download_priority) {
2666 file_id = id;
2667 }
2668 }
2669 }
2670
2671 auto old_priority = node->generate_priority_;
2672 node->set_generate_priority(download_priority, upload_priority);
2673
2674 if (node->generate_priority_ == 0) {
2675 if (old_priority != 0) {
2676 LOG(INFO) << "Cancel file " << file_id << " generation";
2677 do_cancel_generate(node);
2678 }
2679 return;
2680 }
2681
2682 if (old_priority != 0) {
2683 LOG(INFO) << "TODO: change file " << file_id << " generation priority";
2684 return;
2685 }
2686
2687 QueryId id = queries_container_.create(Query{file_id, Query::Type::Generate});
2688 node->generate_id_ = id;
2689 send_closure(file_generate_manager_, &FileGenerateManager::generate_file, id, *node->generate_, node->local_,
2690 node->suggested_path(), [file_manager = this, id] {
2691 class Callback final : public FileGenerateCallback {
2692 ActorId<FileManager> actor_;
2693 uint64 query_id_;
2694
2695 public:
2696 Callback(ActorId<FileManager> actor, QueryId id) : actor_(std::move(actor)), query_id_(id) {
2697 }
2698 void on_partial_generate(PartialLocalFileLocation partial_local, int32 expected_size) final {
2699 send_closure(actor_, &FileManager::on_partial_generate, query_id_, std::move(partial_local),
2700 expected_size);
2701 }
2702 void on_ok(FullLocalFileLocation local) final {
2703 send_closure(actor_, &FileManager::on_generate_ok, query_id_, std::move(local));
2704 }
2705 void on_error(Status error) final {
2706 send_closure(actor_, &FileManager::on_error, query_id_, std::move(error));
2707 }
2708 };
2709 return make_unique<Callback>(file_manager->actor_id(file_manager), id);
2710 }());
2711
2712 LOG(INFO) << "File " << file_id << " generate request has sent to FileGenerateManager";
2713 }
2714
run_upload(FileNodePtr node,std::vector<int> bad_parts)2715 void FileManager::run_upload(FileNodePtr node, std::vector<int> bad_parts) {
2716 int8 priority = 0;
2717 FileId file_id = node->main_file_id_;
2718 for (auto id : node->file_ids_) {
2719 auto *info = get_file_id_info(id);
2720 if (info->upload_priority_ > priority) {
2721 priority = info->upload_priority_;
2722 file_id = id;
2723 }
2724 }
2725
2726 auto old_priority = node->upload_priority_;
2727
2728 if (priority == 0) {
2729 node->set_upload_priority(priority);
2730 if (old_priority != 0) {
2731 LOG(INFO) << "Cancel file " << file_id << " uploading";
2732 do_cancel_upload(node);
2733 }
2734 return;
2735 }
2736
2737 if (node->need_load_from_pmc_) {
2738 LOG(INFO) << "File " << node->main_file_id_ << " needs to be loaded from database before upload";
2739 return;
2740 }
2741 if (node->upload_pause_.is_valid()) {
2742 LOG(INFO) << "File " << node->main_file_id_ << " upload is paused: " << node->upload_pause_;
2743 return;
2744 }
2745
2746 FileView file_view(node);
2747 if (!file_view.has_local_location() && !file_view.has_remote_location()) {
2748 if (node->get_by_hash_ || node->generate_id_ == 0 || !node->generate_was_update_) {
2749 LOG(INFO) << "Have no local location for file: get_by_hash = " << node->get_by_hash_
2750 << ", generate_id = " << node->generate_id_ << ", generate_was_update = " << node->generate_was_update_;
2751 return;
2752 }
2753 if (file_view.has_generate_location() && file_view.generate_location().file_type_ == FileType::Secure) {
2754 // Can't upload secure file before its size is known
2755 LOG(INFO) << "Can't upload secure file " << node->main_file_id_ << " before it's size is known";
2756 return;
2757 }
2758 }
2759
2760 node->set_upload_priority(priority);
2761
2762 // create encryption key if necessary
2763 if (((file_view.has_generate_location() && file_view.generate_location().file_type_ == FileType::Encrypted) ||
2764 (file_view.has_local_location() && file_view.local_location().file_type_ == FileType::Encrypted)) &&
2765 file_view.encryption_key().empty()) {
2766 CHECK(!node->file_ids_.empty());
2767 bool success = set_encryption_key(node->file_ids_[0], FileEncryptionKey::create());
2768 LOG_IF(FATAL, !success) << "Failed to set encryption key for file " << file_id;
2769 }
2770
2771 // create encryption key if necessary
2772 if (file_view.has_local_location() && file_view.local_location().file_type_ == FileType::Secure &&
2773 file_view.encryption_key().empty()) {
2774 CHECK(!node->file_ids_.empty());
2775 bool success = set_encryption_key(node->file_ids_[0], FileEncryptionKey::create_secure_key());
2776 LOG_IF(FATAL, !success) << "Failed to set encryption key for file " << file_id;
2777 }
2778
2779 if (old_priority != 0) {
2780 LOG(INFO) << "File " << file_id << " is already uploading";
2781 CHECK(node->upload_id_ != 0);
2782 send_closure(file_load_manager_, &FileLoadManager::update_priority, node->upload_id_, narrow_cast<int8>(-priority));
2783 return;
2784 }
2785
2786 CHECK(node->upload_id_ == 0);
2787 if (file_view.has_alive_remote_location() && !file_view.has_active_upload_remote_location() &&
2788 file_view.get_type() != FileType::Thumbnail && file_view.get_type() != FileType::EncryptedThumbnail &&
2789 file_view.get_type() != FileType::Background) {
2790 QueryId id = queries_container_.create(Query{file_id, Query::Type::UploadWaitFileReference});
2791 node->upload_id_ = id;
2792 if (node->upload_was_update_file_reference_) {
2793 on_error(id, Status::Error("Can't upload file: have no valid file reference"));
2794 return;
2795 }
2796 node->upload_was_update_file_reference_ = true;
2797
2798 context_->repair_file_reference(
2799 node->main_file_id_, PromiseCreator::lambda([id, actor_id = actor_id(this)](Result<Unit> res) {
2800 send_closure(actor_id, &FileManager::on_error, id, Status::Error("FILE_UPLOAD_RESTART_WITH_FILE_REFERENCE"));
2801 }));
2802 return;
2803 }
2804
2805 if (!node->remote_.partial && node->get_by_hash_) {
2806 LOG(INFO) << "Get file " << node->main_file_id_ << " by hash";
2807 QueryId id = queries_container_.create(Query{file_id, Query::Type::UploadByHash});
2808 node->upload_id_ = id;
2809
2810 send_closure(file_load_manager_, &FileLoadManager::upload_by_hash, id, node->local_.full(), node->size_,
2811 narrow_cast<int8>(-priority));
2812 return;
2813 }
2814
2815 auto new_priority = narrow_cast<int8>(bad_parts.empty() ? -priority : priority);
2816 td::remove_if(bad_parts, [](auto part_id) { return part_id < 0; });
2817
2818 auto expected_size = file_view.expected_size(true);
2819 if (node->upload_prefer_small_ && (10 << 20) < expected_size && expected_size < (30 << 20)) {
2820 expected_size = 10 << 20;
2821 }
2822
2823 QueryId id = queries_container_.create(Query{file_id, Query::Type::Upload});
2824 node->upload_id_ = id;
2825 send_closure(file_load_manager_, &FileLoadManager::upload, id, node->local_, node->remote_.partial_or_empty(),
2826 expected_size, node->encryption_key_, new_priority, std::move(bad_parts));
2827
2828 LOG(INFO) << "File " << file_id << " upload request has sent to FileLoadManager";
2829 }
2830
upload(FileId file_id,std::shared_ptr<UploadCallback> callback,int32 new_priority,uint64 upload_order)2831 void FileManager::upload(FileId file_id, std::shared_ptr<UploadCallback> callback, int32 new_priority,
2832 uint64 upload_order) {
2833 return resume_upload(file_id, std::vector<int>(), std::move(callback), new_priority, upload_order);
2834 }
2835
cancel_upload(FileId file_id)2836 void FileManager::cancel_upload(FileId file_id) {
2837 return resume_upload(file_id, std::vector<int>(), nullptr, 0, 0);
2838 }
2839
is_document_type(FileType type)2840 static bool is_document_type(FileType type) {
2841 return type == FileType::Document || type == FileType::Sticker || type == FileType::Audio ||
2842 type == FileType::Animation || type == FileType::Background || type == FileType::DocumentAsFile;
2843 }
2844
is_background_type(FileType type)2845 static bool is_background_type(FileType type) {
2846 return type == FileType::Wallpaper || type == FileType::Background;
2847 }
2848
from_persistent_id(CSlice persistent_id,FileType file_type)2849 Result<FileId> FileManager::from_persistent_id(CSlice persistent_id, FileType file_type) {
2850 if (persistent_id.find('.') != string::npos) {
2851 auto r_http_url = parse_url(persistent_id);
2852 if (r_http_url.is_error()) {
2853 return Status::Error(400, PSLICE() << "Invalid file HTTP URL specified: " << r_http_url.error().message());
2854 }
2855 auto url = r_http_url.ok().get_url();
2856 if (!clean_input_string(url)) {
2857 return Status::Error(400, "URL must be in UTF-8");
2858 }
2859 return register_url(std::move(url), file_type, FileLocationSource::FromUser, DialogId());
2860 }
2861
2862 auto r_binary = base64url_decode(persistent_id);
2863 if (r_binary.is_error()) {
2864 return Status::Error(400, PSLICE() << "Wrong remote file identifier specified: " << r_binary.error().message());
2865 }
2866 auto binary = r_binary.move_as_ok();
2867 if (binary.empty()) {
2868 return Status::Error(400, "Remote file identifier can't be empty");
2869 }
2870 if (binary.back() == FileNode::PERSISTENT_ID_VERSION_OLD) {
2871 return from_persistent_id_v2(binary, file_type);
2872 }
2873 if (binary.back() == FileNode::PERSISTENT_ID_VERSION) {
2874 return from_persistent_id_v3(binary, file_type);
2875 }
2876 if (binary.back() == FileNode::PERSISTENT_ID_VERSION_MAP) {
2877 return from_persistent_id_map(binary, file_type);
2878 }
2879 return Status::Error(400, "Wrong remote file identifier specified: can't unserialize it. Wrong last symbol");
2880 }
2881
from_persistent_id_map(Slice binary,FileType file_type)2882 Result<FileId> FileManager::from_persistent_id_map(Slice binary, FileType file_type) {
2883 binary.remove_suffix(1);
2884 auto decoded_binary = zero_decode(binary);
2885 FullGenerateFileLocation generate_location;
2886 auto status = unserialize(generate_location, decoded_binary);
2887 if (status.is_error()) {
2888 return Status::Error(400, "Wrong remote file identifier specified: can't unserialize it");
2889 }
2890 auto real_file_type = generate_location.file_type_;
2891 if ((real_file_type != file_type && file_type != FileType::Temp) ||
2892 (real_file_type != FileType::Thumbnail && real_file_type != FileType::EncryptedThumbnail)) {
2893 return Status::Error(400, "Type of file mismatch");
2894 }
2895 if (!begins_with(generate_location.conversion_, "#map#")) {
2896 return Status::Error(400, "Unexpected conversion type");
2897 }
2898 FileData data;
2899 data.generate_ = make_unique<FullGenerateFileLocation>(std::move(generate_location));
2900 return register_file(std::move(data), FileLocationSource::FromUser, "from_persistent_id_map", false).move_as_ok();
2901 }
2902
from_persistent_id_v23(Slice binary,FileType file_type,int32 version)2903 Result<FileId> FileManager::from_persistent_id_v23(Slice binary, FileType file_type, int32 version) {
2904 if (version < 0 || version >= static_cast<int32>(Version::Next)) {
2905 return Status::Error(400, "Invalid remote file identifier");
2906 }
2907 auto decoded_binary = zero_decode(binary);
2908 FullRemoteFileLocation remote_location;
2909 log_event::WithVersion<TlParser> parser(decoded_binary);
2910 parser.set_version(version);
2911 parse(remote_location, parser);
2912 parser.fetch_end();
2913 auto status = parser.get_status();
2914 if (status.is_error()) {
2915 return Status::Error(400, "Wrong remote file identifier specified: can't unserialize it");
2916 }
2917 auto &real_file_type = remote_location.file_type_;
2918 if (is_document_type(real_file_type) && is_document_type(file_type)) {
2919 real_file_type = file_type;
2920 } else if (is_background_type(real_file_type) && is_background_type(file_type)) {
2921 // type of file matches, but real type is in the stored remote location
2922 } else if (real_file_type != file_type && file_type != FileType::Temp) {
2923 return Status::Error(400, "Type of file mismatch");
2924 }
2925 FileData data;
2926 data.remote_ = RemoteFileLocation(std::move(remote_location));
2927 auto file_id =
2928 register_file(std::move(data), FileLocationSource::FromUser, "from_persistent_id_v23", false).move_as_ok();
2929 return file_id;
2930 }
2931
from_persistent_id_v2(Slice binary,FileType file_type)2932 Result<FileId> FileManager::from_persistent_id_v2(Slice binary, FileType file_type) {
2933 binary.remove_suffix(1);
2934 return from_persistent_id_v23(binary, file_type, 0);
2935 }
2936
from_persistent_id_v3(Slice binary,FileType file_type)2937 Result<FileId> FileManager::from_persistent_id_v3(Slice binary, FileType file_type) {
2938 binary.remove_suffix(1);
2939 if (binary.empty()) {
2940 return Status::Error(400, "Invalid remote file identifier");
2941 }
2942 int32 version = static_cast<uint8>(binary.back());
2943 binary.remove_suffix(1);
2944 return from_persistent_id_v23(binary, file_type, version);
2945 }
2946
get_file_view(FileId file_id) const2947 FileView FileManager::get_file_view(FileId file_id) const {
2948 auto file_node = get_file_node(file_id);
2949 if (!file_node) {
2950 return FileView();
2951 }
2952 return FileView(file_node);
2953 }
2954
get_sync_file_view(FileId file_id)2955 FileView FileManager::get_sync_file_view(FileId file_id) {
2956 auto file_node = get_sync_file_node(file_id);
2957 if (!file_node) {
2958 return FileView();
2959 }
2960 return FileView(file_node);
2961 }
2962
get_file_object(FileId file_id,bool with_main_file_id)2963 td_api::object_ptr<td_api::file> FileManager::get_file_object(FileId file_id, bool with_main_file_id) {
2964 auto file_view = get_sync_file_view(file_id);
2965
2966 if (file_view.empty()) {
2967 return td_api::make_object<td_api::file>(0, 0, 0, td_api::make_object<td_api::localFile>(),
2968 td_api::make_object<td_api::remoteFile>());
2969 }
2970
2971 string persistent_file_id = file_view.get_persistent_file_id();
2972 string unique_file_id = file_view.get_unique_file_id();
2973 bool is_uploading_completed = !persistent_file_id.empty();
2974 auto size = narrow_cast<int32>(file_view.size());
2975 auto expected_size = narrow_cast<int32>(file_view.expected_size());
2976 auto download_offset = narrow_cast<int32>(file_view.download_offset());
2977 auto local_prefix_size = narrow_cast<int32>(file_view.local_prefix_size());
2978 auto local_total_size = narrow_cast<int32>(file_view.local_total_size());
2979 auto remote_size = narrow_cast<int32>(file_view.remote_size());
2980 string path = file_view.path();
2981 bool can_be_downloaded = file_view.can_download_from_server() || file_view.can_generate();
2982 bool can_be_deleted = file_view.can_delete();
2983
2984 auto result_file_id = file_id;
2985 auto *file_info = get_file_id_info(result_file_id);
2986 if (with_main_file_id) {
2987 if (!file_info->send_updates_flag_) {
2988 result_file_id = file_view.file_id();
2989 }
2990 file_info = get_file_id_info(file_view.file_id());
2991 }
2992 file_info->send_updates_flag_ = true;
2993 VLOG(update_file) << "Send file " << file_id << " as " << result_file_id << " and update send_updates_flag_ for file "
2994 << (with_main_file_id ? file_view.file_id() : result_file_id);
2995
2996 return td_api::make_object<td_api::file>(
2997 result_file_id.get(), size, expected_size,
2998 td_api::make_object<td_api::localFile>(std::move(path), can_be_downloaded, can_be_deleted,
2999 file_view.is_downloading(), file_view.has_local_location(),
3000 download_offset, local_prefix_size, local_total_size),
3001 td_api::make_object<td_api::remoteFile>(std::move(persistent_file_id), std::move(unique_file_id),
3002 file_view.is_uploading(), is_uploading_completed, remote_size));
3003 }
3004
get_file_ids_object(const vector<FileId> & file_ids,bool with_main_file_id)3005 vector<int32> FileManager::get_file_ids_object(const vector<FileId> &file_ids, bool with_main_file_id) {
3006 return transform(file_ids, [this, with_main_file_id](FileId file_id) {
3007 auto file_view = get_sync_file_view(file_id);
3008 auto result_file_id = file_id;
3009 auto *file_info = get_file_id_info(result_file_id);
3010 if (with_main_file_id) {
3011 if (!file_info->sent_file_id_flag_ && !file_info->send_updates_flag_) {
3012 result_file_id = file_view.file_id();
3013 }
3014 file_info = get_file_id_info(file_view.file_id());
3015 }
3016 file_info->sent_file_id_flag_ = true;
3017
3018 return result_file_id.get();
3019 });
3020 }
3021
check_input_file_id(FileType type,Result<FileId> result,bool is_encrypted,bool allow_zero,bool is_secure)3022 Result<FileId> FileManager::check_input_file_id(FileType type, Result<FileId> result, bool is_encrypted,
3023 bool allow_zero, bool is_secure) {
3024 TRY_RESULT(file_id, std::move(result));
3025 if (allow_zero && !file_id.is_valid()) {
3026 return FileId();
3027 }
3028
3029 auto file_node = get_sync_file_node(file_id); // we need full data about sent files
3030 if (!file_node) {
3031 return Status::Error(400, "File not found");
3032 }
3033 auto file_view = FileView(file_node);
3034 FileType real_type = file_view.get_type();
3035 LOG(INFO) << "Checking file " << file_id << " of type " << type << "/" << real_type;
3036 if (!is_encrypted && !is_secure) {
3037 if (real_type != type && !(real_type == FileType::Temp && file_view.has_url()) &&
3038 !(is_document_type(real_type) && is_document_type(type)) &&
3039 !(is_background_type(real_type) && is_background_type(type))) {
3040 // TODO: send encrypted file to unencrypted chat
3041 return Status::Error(400, "Type of file mismatch");
3042 }
3043 }
3044
3045 if (!file_view.has_remote_location()) {
3046 // TODO why not return file_id here? We will dup it anyway
3047 // But it will not be duped if has_input_media(), so for now we can't return main_file_id
3048
3049 if (file_view.has_url() && !is_encrypted) {
3050 // URLs in non-secret chats never needs to be reuploaded, so they don't need to be duped
3051 return file_node->main_file_id_;
3052 }
3053 return dup_file_id(file_id);
3054 }
3055
3056 int32 remote_id = file_id.get_remote();
3057 if (remote_id == 0) {
3058 RemoteInfo info{file_view.remote_location(), FileLocationSource::FromUser, file_id};
3059 remote_id = remote_location_info_.add(info);
3060 if (remote_location_info_.get(remote_id).file_id_ == file_id) {
3061 get_file_id_info(file_id)->pin_flag_ = true;
3062 }
3063 }
3064 return FileId(file_node->main_file_id_.get(), remote_id);
3065 }
3066
get_input_thumbnail_file_id(const tl_object_ptr<td_api::InputFile> & thumbnail_input_file,DialogId owner_dialog_id,bool is_encrypted)3067 Result<FileId> FileManager::get_input_thumbnail_file_id(const tl_object_ptr<td_api::InputFile> &thumbnail_input_file,
3068 DialogId owner_dialog_id, bool is_encrypted) {
3069 if (thumbnail_input_file == nullptr) {
3070 return Status::Error(400, "inputThumbnail not specified");
3071 }
3072
3073 switch (thumbnail_input_file->get_id()) {
3074 case td_api::inputFileLocal::ID: {
3075 const string &path = static_cast<const td_api::inputFileLocal *>(thumbnail_input_file.get())->path_;
3076 return register_local(
3077 FullLocalFileLocation(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail, path, 0),
3078 owner_dialog_id, 0, false);
3079 }
3080 case td_api::inputFileId::ID:
3081 return Status::Error(400, "InputFileId is not supported for thumbnails");
3082 case td_api::inputFileRemote::ID:
3083 return Status::Error(400, "InputFileRemote is not supported for thumbnails");
3084 case td_api::inputFileGenerated::ID: {
3085 auto *generated_thumbnail = static_cast<const td_api::inputFileGenerated *>(thumbnail_input_file.get());
3086 return register_generate(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail,
3087 FileLocationSource::FromUser, generated_thumbnail->original_path_,
3088 generated_thumbnail->conversion_, owner_dialog_id, generated_thumbnail->expected_size_);
3089 }
3090 default:
3091 UNREACHABLE();
3092 return Status::Error(500, "Unreachable");
3093 }
3094 }
3095
get_input_file_id(FileType type,const tl_object_ptr<td_api::InputFile> & file,DialogId owner_dialog_id,bool allow_zero,bool is_encrypted,bool get_by_hash,bool is_secure)3096 Result<FileId> FileManager::get_input_file_id(FileType type, const tl_object_ptr<td_api::InputFile> &file,
3097 DialogId owner_dialog_id, bool allow_zero, bool is_encrypted,
3098 bool get_by_hash, bool is_secure) {
3099 if (file == nullptr) {
3100 if (allow_zero) {
3101 return FileId();
3102 }
3103 return Status::Error(400, "InputFile is not specified");
3104 }
3105
3106 if (is_encrypted || is_secure) {
3107 get_by_hash = false;
3108 }
3109
3110 auto new_type = is_encrypted ? FileType::Encrypted : (is_secure ? FileType::Secure : type);
3111
3112 auto r_file_id = [&]() -> Result<FileId> {
3113 switch (file->get_id()) {
3114 case td_api::inputFileLocal::ID: {
3115 const string &path = static_cast<const td_api::inputFileLocal *>(file.get())->path_;
3116 if (allow_zero && path.empty()) {
3117 return FileId();
3118 }
3119 string hash;
3120 if (G()->shared_config().get_option_boolean("reuse_uploaded_photos_by_hash") && new_type == FileType::Photo) {
3121 auto r_stat = stat(path);
3122 if (r_stat.is_ok() && r_stat.ok().size_ > 0 && r_stat.ok().size_ < 11000000) {
3123 auto r_file_content = read_file_str(path, r_stat.ok().size_);
3124 if (r_file_content.is_ok()) {
3125 hash = sha256(r_file_content.ok());
3126 auto it = file_hash_to_file_id_.find(hash);
3127 if (it != file_hash_to_file_id_.end()) {
3128 auto file_view = get_file_view(it->second);
3129 if (!file_view.empty() && file_view.has_remote_location() && !file_view.remote_location().is_web()) {
3130 return it->second;
3131 }
3132 }
3133 }
3134 }
3135 }
3136 TRY_RESULT(file_id, register_local(FullLocalFileLocation(new_type, path, 0), owner_dialog_id, 0, get_by_hash));
3137 if (!hash.empty()) {
3138 file_hash_to_file_id_[hash] = file_id;
3139 }
3140 return file_id;
3141 }
3142 case td_api::inputFileId::ID: {
3143 FileId file_id(static_cast<const td_api::inputFileId *>(file.get())->id_, 0);
3144 if (!file_id.is_valid()) {
3145 return FileId();
3146 }
3147 return file_id;
3148 }
3149 case td_api::inputFileRemote::ID: {
3150 const string &file_persistent_id = static_cast<const td_api::inputFileRemote *>(file.get())->id_;
3151 if (allow_zero && file_persistent_id.empty()) {
3152 return FileId();
3153 }
3154 return from_persistent_id(file_persistent_id, type);
3155 }
3156 case td_api::inputFileGenerated::ID: {
3157 auto *generated_file = static_cast<const td_api::inputFileGenerated *>(file.get());
3158 return register_generate(new_type, FileLocationSource::FromUser, generated_file->original_path_,
3159 generated_file->conversion_, owner_dialog_id, generated_file->expected_size_);
3160 }
3161 default:
3162 UNREACHABLE();
3163 return Status::Error(500, "Unreachable");
3164 }
3165 }();
3166
3167 return check_input_file_id(type, std::move(r_file_id), is_encrypted, allow_zero, is_secure);
3168 }
3169
get_map_thumbnail_file_id(Location location,int32 zoom,int32 width,int32 height,int32 scale,DialogId owner_dialog_id)3170 Result<FileId> FileManager::get_map_thumbnail_file_id(Location location, int32 zoom, int32 width, int32 height,
3171 int32 scale, DialogId owner_dialog_id) {
3172 if (!location.is_valid_map_point()) {
3173 return Status::Error(400, "Invalid location specified");
3174 }
3175 if (zoom < 13 || zoom > 20) {
3176 return Status::Error(400, "Wrong zoom");
3177 }
3178 if (width < 16 || width > 1024) {
3179 return Status::Error(400, "Wrong width");
3180 }
3181 if (height < 16 || height > 1024) {
3182 return Status::Error(400, "Wrong height");
3183 }
3184 if (scale < 1 || scale > 3) {
3185 return Status::Error(400, "Wrong scale");
3186 }
3187
3188 const double PI = 3.14159265358979323846;
3189 double sin_latitude = std::sin(location.get_latitude() * PI / 180);
3190 int32 size = 256 * (1 << zoom);
3191 auto x = static_cast<int32>((location.get_longitude() + 180) / 360 * size);
3192 auto y = static_cast<int32>((0.5 - std::log((1 + sin_latitude) / (1 - sin_latitude)) / (4 * PI)) * size);
3193 x = clamp(x, 0, size - 1); // just in case
3194 y = clamp(y, 0, size - 1); // just in case
3195
3196 string conversion = PSTRING() << "#map#" << zoom << '#' << x << '#' << y << '#' << width << '#' << height << '#'
3197 << scale << '#';
3198 return register_generate(
3199 owner_dialog_id.get_type() == DialogType::SecretChat ? FileType::EncryptedThumbnail : FileType::Thumbnail,
3200 FileLocationSource::FromUser, string(), std::move(conversion), owner_dialog_id, 0);
3201 }
3202
guess_file_type(const tl_object_ptr<td_api::InputFile> & file)3203 FileType FileManager::guess_file_type(const tl_object_ptr<td_api::InputFile> &file) {
3204 if (file == nullptr) {
3205 return FileType::Temp;
3206 }
3207
3208 auto guess_file_type_by_path = [](const string &file_path) {
3209 PathView path_view(file_path);
3210 auto file_name = path_view.file_name();
3211 auto extension = path_view.extension();
3212 if (extension == "jpg" || extension == "jpeg") {
3213 return FileType::Photo;
3214 }
3215 if (extension == "ogg" || extension == "oga" || extension == "opus") {
3216 return FileType::VoiceNote;
3217 }
3218 if (extension == "3gp" || extension == "mov") {
3219 return FileType::Video;
3220 }
3221 if (extension == "mp3" || extension == "mpeg3" || extension == "m4a") {
3222 return FileType::Audio;
3223 }
3224 if (extension == "webp" || extension == "tgs") {
3225 return FileType::Sticker;
3226 }
3227 if (extension == "gif") {
3228 return FileType::Animation;
3229 }
3230 if (extension == "mp4" || extension == "mpeg4") {
3231 return to_lower(file_name).find("-gif-") != string::npos ? FileType::Animation : FileType::Video;
3232 }
3233 return FileType::Document;
3234 };
3235
3236 switch (file->get_id()) {
3237 case td_api::inputFileLocal::ID:
3238 return guess_file_type_by_path(static_cast<const td_api::inputFileLocal *>(file.get())->path_);
3239 case td_api::inputFileId::ID: {
3240 FileId file_id(static_cast<const td_api::inputFileId *>(file.get())->id_, 0);
3241 auto file_view = get_file_view(file_id);
3242 if (file_view.empty()) {
3243 return FileType::Temp;
3244 }
3245 return file_view.get_type();
3246 }
3247 case td_api::inputFileRemote::ID: {
3248 const string &file_persistent_id = static_cast<const td_api::inputFileRemote *>(file.get())->id_;
3249 Result<FileId> r_file_id = from_persistent_id(file_persistent_id, FileType::Temp);
3250 if (r_file_id.is_error()) {
3251 return FileType::Temp;
3252 }
3253 auto file_view = get_file_view(r_file_id.ok());
3254 if (file_view.empty()) {
3255 return FileType::Temp;
3256 }
3257 return file_view.get_type();
3258 }
3259 case td_api::inputFileGenerated::ID:
3260 return guess_file_type_by_path(static_cast<const td_api::inputFileGenerated *>(file.get())->original_path_);
3261 default:
3262 UNREACHABLE();
3263 return FileType::Temp;
3264 }
3265 }
3266
get_input_documents(const vector<FileId> & file_ids)3267 vector<tl_object_ptr<telegram_api::InputDocument>> FileManager::get_input_documents(const vector<FileId> &file_ids) {
3268 vector<tl_object_ptr<telegram_api::InputDocument>> result;
3269 result.reserve(file_ids.size());
3270 for (auto file_id : file_ids) {
3271 auto file_view = get_file_view(file_id);
3272 CHECK(!file_view.empty());
3273 CHECK(file_view.has_remote_location());
3274 CHECK(!file_view.remote_location().is_web());
3275 result.push_back(file_view.remote_location().as_input_document());
3276 }
3277 return result;
3278 }
3279
extract_was_uploaded(const tl_object_ptr<telegram_api::InputMedia> & input_media)3280 bool FileManager::extract_was_uploaded(const tl_object_ptr<telegram_api::InputMedia> &input_media) {
3281 if (input_media == nullptr) {
3282 return false;
3283 }
3284
3285 auto input_media_id = input_media->get_id();
3286 return input_media_id == telegram_api::inputMediaUploadedPhoto::ID ||
3287 input_media_id == telegram_api::inputMediaUploadedDocument::ID;
3288 }
3289
extract_was_thumbnail_uploaded(const tl_object_ptr<telegram_api::InputMedia> & input_media)3290 bool FileManager::extract_was_thumbnail_uploaded(const tl_object_ptr<telegram_api::InputMedia> &input_media) {
3291 if (input_media == nullptr || input_media->get_id() != telegram_api::inputMediaUploadedDocument::ID) {
3292 return false;
3293 }
3294
3295 return static_cast<const telegram_api::inputMediaUploadedDocument *>(input_media.get())->thumb_ != nullptr;
3296 }
3297
extract_file_reference(const tl_object_ptr<telegram_api::InputMedia> & input_media)3298 string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputMedia> &input_media) {
3299 if (input_media == nullptr) {
3300 return string();
3301 }
3302
3303 switch (input_media->get_id()) {
3304 case telegram_api::inputMediaDocument::ID:
3305 return extract_file_reference(static_cast<const telegram_api::inputMediaDocument *>(input_media.get())->id_);
3306 case telegram_api::inputMediaPhoto::ID:
3307 return extract_file_reference(static_cast<const telegram_api::inputMediaPhoto *>(input_media.get())->id_);
3308 default:
3309 return string();
3310 }
3311 }
3312
extract_file_reference(const tl_object_ptr<telegram_api::InputDocument> & input_document)3313 string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputDocument> &input_document) {
3314 if (input_document == nullptr || input_document->get_id() != telegram_api::inputDocument::ID) {
3315 return string();
3316 }
3317
3318 return static_cast<const telegram_api::inputDocument *>(input_document.get())->file_reference_.as_slice().str();
3319 }
3320
extract_file_reference(const tl_object_ptr<telegram_api::InputPhoto> & input_photo)3321 string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputPhoto> &input_photo) {
3322 if (input_photo == nullptr || input_photo->get_id() != telegram_api::inputPhoto::ID) {
3323 return string();
3324 }
3325
3326 return static_cast<const telegram_api::inputPhoto *>(input_photo.get())->file_reference_.as_slice().str();
3327 }
3328
extract_was_uploaded(const tl_object_ptr<telegram_api::InputChatPhoto> & input_chat_photo)3329 bool FileManager::extract_was_uploaded(const tl_object_ptr<telegram_api::InputChatPhoto> &input_chat_photo) {
3330 return input_chat_photo != nullptr && input_chat_photo->get_id() == telegram_api::inputChatUploadedPhoto::ID;
3331 }
3332
extract_file_reference(const tl_object_ptr<telegram_api::InputChatPhoto> & input_chat_photo)3333 string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputChatPhoto> &input_chat_photo) {
3334 if (input_chat_photo == nullptr || input_chat_photo->get_id() != telegram_api::inputChatPhoto::ID) {
3335 return string();
3336 }
3337
3338 return extract_file_reference(static_cast<const telegram_api::inputChatPhoto *>(input_chat_photo.get())->id_);
3339 }
3340
next_file_id()3341 FileId FileManager::next_file_id() {
3342 if (!empty_file_ids_.empty()) {
3343 auto res = empty_file_ids_.back();
3344 empty_file_ids_.pop_back();
3345 return FileId{res, 0};
3346 }
3347 FileId res(static_cast<int32>(file_id_info_.size()), 0);
3348 // LOG(ERROR) << "NEXT file_id " << res;
3349 file_id_info_.push_back({});
3350 return res;
3351 }
3352
next_file_node_id()3353 FileManager::FileNodeId FileManager::next_file_node_id() {
3354 auto res = static_cast<FileNodeId>(file_nodes_.size());
3355 file_nodes_.emplace_back(nullptr);
3356 return res;
3357 }
3358
on_start_download(QueryId query_id)3359 void FileManager::on_start_download(QueryId query_id) {
3360 if (is_closed_) {
3361 return;
3362 }
3363
3364 auto query = queries_container_.get(query_id);
3365 CHECK(query != nullptr);
3366
3367 auto file_id = query->file_id_;
3368 auto file_node = get_file_node(file_id);
3369 LOG(DEBUG) << "Receive on_start_download for file " << file_id;
3370 if (!file_node) {
3371 return;
3372 }
3373 if (file_node->download_id_ != query_id) {
3374 return;
3375 }
3376
3377 LOG(DEBUG) << "Start to download part of file " << file_id;
3378 file_node->is_download_started_ = true;
3379 }
3380
on_partial_download(QueryId query_id,PartialLocalFileLocation partial_local,int64 ready_size,int64 size)3381 void FileManager::on_partial_download(QueryId query_id, PartialLocalFileLocation partial_local, int64 ready_size,
3382 int64 size) {
3383 if (is_closed_) {
3384 return;
3385 }
3386
3387 auto query = queries_container_.get(query_id);
3388 CHECK(query != nullptr);
3389
3390 auto file_id = query->file_id_;
3391 auto file_node = get_file_node(file_id);
3392 LOG(DEBUG) << "Receive on_partial_download for file " << file_id << " with " << partial_local
3393 << ", ready_size = " << ready_size << " and size = " << size;
3394 if (!file_node) {
3395 return;
3396 }
3397 if (file_node->download_id_ != query_id) {
3398 return;
3399 }
3400
3401 if (size != 0) {
3402 FileView file_view(file_node);
3403 if (!file_view.is_encrypted_secure()) {
3404 file_node->set_size(size);
3405 }
3406 }
3407 file_node->set_local_location(LocalFileLocation(std::move(partial_local)), ready_size, -1, -1 /* TODO */);
3408 try_flush_node(file_node, "on_partial_download");
3409 }
3410
on_hash(QueryId query_id,string hash)3411 void FileManager::on_hash(QueryId query_id, string hash) {
3412 if (is_closed_) {
3413 return;
3414 }
3415
3416 auto query = queries_container_.get(query_id);
3417 CHECK(query != nullptr);
3418
3419 auto file_id = query->file_id_;
3420
3421 auto file_node = get_file_node(file_id);
3422 LOG(DEBUG) << "Receive on_hash for file " << file_id;
3423 if (!file_node) {
3424 return;
3425 }
3426 if (file_node->upload_id_ != query_id) {
3427 return;
3428 }
3429
3430 file_node->encryption_key_.set_value_hash(secure_storage::ValueHash::create(hash).move_as_ok());
3431 }
3432
on_partial_upload(QueryId query_id,PartialRemoteFileLocation partial_remote,int64 ready_size)3433 void FileManager::on_partial_upload(QueryId query_id, PartialRemoteFileLocation partial_remote, int64 ready_size) {
3434 if (is_closed_) {
3435 return;
3436 }
3437
3438 auto query = queries_container_.get(query_id);
3439 CHECK(query != nullptr);
3440
3441 auto file_id = query->file_id_;
3442 auto file_node = get_file_node(file_id);
3443 LOG(DEBUG) << "Receive on_partial_upload for file " << file_id << " with " << partial_remote << " and ready size "
3444 << ready_size;
3445 if (!file_node) {
3446 return;
3447 }
3448 if (file_node->upload_id_ != query_id) {
3449 return;
3450 }
3451
3452 file_node->set_partial_remote_location(std::move(partial_remote), ready_size);
3453 try_flush_node(file_node, "on_partial_upload");
3454 }
3455
on_download_ok(QueryId query_id,FullLocalFileLocation local,int64 size,bool is_new)3456 void FileManager::on_download_ok(QueryId query_id, FullLocalFileLocation local, int64 size, bool is_new) {
3457 if (is_closed_) {
3458 return;
3459 }
3460
3461 Query query;
3462 bool was_active;
3463 std::tie(query, was_active) = finish_query(query_id);
3464 auto file_id = query.file_id_;
3465 LOG(INFO) << "ON DOWNLOAD OK of " << (is_new ? "new" : "checked") << " file " << file_id << " of size " << size;
3466 auto r_new_file_id = register_local(std::move(local), DialogId(), size, false, false, true);
3467 Status status = Status::OK();
3468 if (r_new_file_id.is_error()) {
3469 status = Status::Error(PSLICE() << "Can't register local file after download: " << r_new_file_id.error().message());
3470 } else {
3471 if (is_new) {
3472 context_->on_new_file(size, get_file_view(r_new_file_id.ok()).get_allocated_local_size(), 1);
3473 }
3474 auto r_file_id = merge(r_new_file_id.ok(), file_id);
3475 if (r_file_id.is_error()) {
3476 status = r_file_id.move_as_error();
3477 }
3478 }
3479 if (status.is_error()) {
3480 LOG(ERROR) << status.message();
3481 return on_error_impl(get_file_node(file_id), query.type_, was_active, std::move(status));
3482 }
3483 }
3484
on_upload_ok(QueryId query_id,FileType file_type,PartialRemoteFileLocation partial_remote,int64 size)3485 void FileManager::on_upload_ok(QueryId query_id, FileType file_type, PartialRemoteFileLocation partial_remote,
3486 int64 size) {
3487 if (is_closed_) {
3488 return;
3489 }
3490
3491 CHECK(partial_remote.ready_part_count_ == partial_remote.part_count_);
3492 auto some_file_id = finish_query(query_id).first.file_id_;
3493 LOG(INFO) << "ON UPLOAD OK file " << some_file_id << " of size " << size;
3494
3495 auto file_node = get_file_node(some_file_id);
3496 if (!file_node) {
3497 return;
3498 }
3499
3500 FileId file_id;
3501 uint64 file_id_upload_order{std::numeric_limits<uint64>::max()};
3502 for (auto id : file_node->file_ids_) {
3503 auto *info = get_file_id_info(id);
3504 if (info->upload_priority_ != 0 && info->upload_order_ < file_id_upload_order) {
3505 file_id = id;
3506 file_id_upload_order = info->upload_order_;
3507 }
3508 }
3509 if (!file_id.is_valid()) {
3510 return;
3511 }
3512
3513 auto *file_info = get_file_id_info(file_id);
3514 LOG(INFO) << "Found being uploaded file " << file_id << " with priority " << file_info->upload_priority_;
3515 file_info->upload_priority_ = 0;
3516 file_info->download_priority_ = 0;
3517
3518 FileView file_view(file_node);
3519 string file_name = get_file_name(file_type, file_view.suggested_path());
3520
3521 if (file_view.is_encrypted_secret()) {
3522 tl_object_ptr<telegram_api::InputEncryptedFile> input_file;
3523 if (partial_remote.is_big_) {
3524 input_file = make_tl_object<telegram_api::inputEncryptedFileBigUploaded>(
3525 partial_remote.file_id_, partial_remote.part_count_, file_view.encryption_key().calc_fingerprint());
3526 } else {
3527 input_file = make_tl_object<telegram_api::inputEncryptedFileUploaded>(
3528 partial_remote.file_id_, partial_remote.part_count_, "", file_view.encryption_key().calc_fingerprint());
3529 }
3530 if (file_info->upload_callback_) {
3531 file_info->upload_callback_->on_upload_encrypted_ok(file_id, std::move(input_file));
3532 file_node->set_upload_pause(file_id);
3533 file_info->upload_callback_.reset();
3534 }
3535 } else if (file_view.is_secure()) {
3536 tl_object_ptr<telegram_api::InputSecureFile> input_file;
3537 input_file = make_tl_object<telegram_api::inputSecureFileUploaded>(
3538 partial_remote.file_id_, partial_remote.part_count_, "" /*md5*/, BufferSlice() /*file_hash*/,
3539 BufferSlice() /*encrypted_secret*/);
3540 if (file_info->upload_callback_) {
3541 file_info->upload_callback_->on_upload_secure_ok(file_id, std::move(input_file));
3542 file_node->upload_pause_ = file_id;
3543 file_info->upload_callback_.reset();
3544 }
3545 } else {
3546 tl_object_ptr<telegram_api::InputFile> input_file;
3547 if (partial_remote.is_big_) {
3548 input_file = make_tl_object<telegram_api::inputFileBig>(partial_remote.file_id_, partial_remote.part_count_,
3549 std::move(file_name));
3550 } else {
3551 input_file = make_tl_object<telegram_api::inputFile>(partial_remote.file_id_, partial_remote.part_count_,
3552 std::move(file_name), "");
3553 }
3554 if (file_info->upload_callback_) {
3555 file_info->upload_callback_->on_upload_ok(file_id, std::move(input_file));
3556 file_node->set_upload_pause(file_id);
3557 file_info->upload_callback_.reset();
3558 }
3559 }
3560 }
3561
on_upload_full_ok(QueryId query_id,FullRemoteFileLocation remote)3562 void FileManager::on_upload_full_ok(QueryId query_id, FullRemoteFileLocation remote) {
3563 if (is_closed_) {
3564 return;
3565 }
3566
3567 auto file_id = finish_query(query_id).first.file_id_;
3568 LOG(INFO) << "ON UPLOAD FULL OK for file " << file_id;
3569 auto new_file_id = register_remote(std::move(remote), FileLocationSource::FromServer, DialogId(), 0, 0, "");
3570 LOG_STATUS(merge(new_file_id, file_id));
3571 }
3572
on_partial_generate(QueryId query_id,PartialLocalFileLocation partial_local,int32 expected_size)3573 void FileManager::on_partial_generate(QueryId query_id, PartialLocalFileLocation partial_local, int32 expected_size) {
3574 if (is_closed_) {
3575 return;
3576 }
3577
3578 auto query = queries_container_.get(query_id);
3579 CHECK(query != nullptr);
3580
3581 auto file_id = query->file_id_;
3582 auto file_node = get_file_node(file_id);
3583 auto bitmask = Bitmask(Bitmask::Decode{}, partial_local.ready_bitmask_);
3584 LOG(DEBUG) << "Receive on_partial_generate for file " << file_id << ": " << partial_local.path_ << " " << bitmask;
3585 if (!file_node) {
3586 return;
3587 }
3588 if (file_node->generate_id_ != query_id) {
3589 return;
3590 }
3591 auto ready_size = bitmask.get_total_size(partial_local.part_size_, file_node->size_);
3592 file_node->set_local_location(LocalFileLocation(partial_local), ready_size, -1, -1 /* TODO */);
3593 // TODO check for size and local_size, abort generation if needed
3594 if (expected_size > 0) {
3595 file_node->set_expected_size(expected_size);
3596 }
3597 if (!file_node->generate_was_update_) {
3598 file_node->generate_was_update_ = true;
3599 run_upload(file_node, {});
3600 }
3601 if (file_node->upload_id_ != 0) {
3602 send_closure(file_load_manager_, &FileLoadManager::update_local_file_location, file_node->upload_id_,
3603 LocalFileLocation(std::move(partial_local)));
3604 }
3605
3606 try_flush_node(file_node, "on_partial_generate");
3607 }
3608
on_generate_ok(QueryId query_id,FullLocalFileLocation local)3609 void FileManager::on_generate_ok(QueryId query_id, FullLocalFileLocation local) {
3610 if (is_closed_) {
3611 return;
3612 }
3613
3614 Query query;
3615 bool was_active;
3616 std::tie(query, was_active) = finish_query(query_id);
3617 auto generate_file_id = query.file_id_;
3618
3619 LOG(INFO) << "Receive on_generate_ok for file " << generate_file_id << ": " << local;
3620 auto file_node = get_file_node(generate_file_id);
3621 if (!file_node) {
3622 return;
3623 }
3624
3625 auto old_upload_id = file_node->upload_id_;
3626
3627 auto r_new_file_id = register_local(local, DialogId(), 0);
3628 Status status;
3629 if (r_new_file_id.is_error()) {
3630 status = Status::Error(PSLICE() << "Can't register local file after generate: " << r_new_file_id.error());
3631 } else {
3632 auto result = merge(r_new_file_id.ok(), generate_file_id);
3633 if (result.is_error()) {
3634 status = result.move_as_error();
3635 }
3636 }
3637 file_node = get_file_node(generate_file_id);
3638 if (status.is_error()) {
3639 return on_error_impl(file_node, query.type_, was_active, std::move(status));
3640 }
3641 CHECK(file_node);
3642
3643 FileView file_view(file_node);
3644 if (!file_view.has_generate_location() || !begins_with(file_view.generate_location().conversion_, "#file_id#")) {
3645 context_->on_new_file(file_view.size(), file_view.get_allocated_local_size(), 1);
3646 }
3647
3648 run_upload(file_node, {});
3649
3650 if (was_active) {
3651 if (old_upload_id != 0 && old_upload_id == file_node->upload_id_) {
3652 send_closure(file_load_manager_, &FileLoadManager::update_local_file_location, file_node->upload_id_,
3653 LocalFileLocation(std::move(local)));
3654 }
3655 }
3656 }
3657
on_error(QueryId query_id,Status status)3658 void FileManager::on_error(QueryId query_id, Status status) {
3659 if (is_closed_) {
3660 return;
3661 }
3662
3663 Query query;
3664 bool was_active;
3665 std::tie(query, was_active) = finish_query(query_id);
3666 auto node = get_file_node(query.file_id_);
3667 if (!node) {
3668 LOG(ERROR) << "Can't find file node for " << query.file_id_ << " " << status;
3669 return;
3670 }
3671
3672 if (query.type_ == Query::Type::UploadByHash && !G()->close_flag()) {
3673 LOG(INFO) << "Upload By Hash failed: " << status << ", restart upload";
3674 node->get_by_hash_ = false;
3675 run_upload(node, {});
3676 return;
3677 }
3678 on_error_impl(node, query.type_, was_active, std::move(status));
3679 }
3680
on_error_impl(FileNodePtr node,Query::Type type,bool was_active,Status status)3681 void FileManager::on_error_impl(FileNodePtr node, Query::Type type, bool was_active, Status status) {
3682 SCOPE_EXIT {
3683 try_flush_node(node, "on_error");
3684 };
3685 if (status.code() != 1 && !G()->close_flag()) {
3686 LOG(WARNING) << "Failed to " << type << " file " << node->main_file_id_ << " of type " << FileView(node).get_type()
3687 << ": " << status;
3688 if (status.code() == 0) {
3689 // Remove partial locations
3690 if (node->local_.type() == LocalFileLocation::Type::Partial &&
3691 !begins_with(status.message(), "FILE_UPLOAD_RESTART") &&
3692 !begins_with(status.message(), "FILE_DOWNLOAD_RESTART") &&
3693 !begins_with(status.message(), "FILE_DOWNLOAD_ID_INVALID") &&
3694 !begins_with(status.message(), "FILE_DOWNLOAD_LIMIT")) {
3695 CSlice path = node->local_.partial().path_;
3696 if (begins_with(path, get_files_temp_dir(FileType::Encrypted)) ||
3697 begins_with(path, get_files_temp_dir(FileType::Video))) {
3698 LOG(INFO) << "Unlink file " << path;
3699 unlink(path).ignore();
3700 node->drop_local_location();
3701 }
3702 }
3703 node->delete_partial_remote_location();
3704 status = Status::Error(400, status.message());
3705 }
3706 }
3707
3708 if (status.message() == "FILE_PART_INVALID") {
3709 bool has_partial_small_location = node->remote_.partial && !node->remote_.partial->is_big_;
3710 FileView file_view(node);
3711 auto expected_size = file_view.expected_size(true);
3712 bool should_be_big_location = is_file_big(file_view.get_type(), expected_size);
3713
3714 node->delete_partial_remote_location();
3715 if (has_partial_small_location && should_be_big_location) {
3716 run_upload(node, {});
3717 return;
3718 }
3719
3720 LOG(WARNING) << "Failed to upload file " << node->main_file_id_ << ": unexpected " << status
3721 << ", is_small = " << has_partial_small_location << ", should_be_big = " << should_be_big_location
3722 << ", expected size = " << expected_size;
3723 }
3724
3725 if (begins_with(status.message(), "FILE_GENERATE_LOCATION_INVALID")) {
3726 node->set_generate_location(nullptr);
3727 }
3728
3729 if ((status.message() == "FILE_ID_INVALID" || status.message() == "LOCATION_INVALID") &&
3730 FileView(node).may_reload_photo()) {
3731 node->need_reload_photo_ = true;
3732 run_download(node, true);
3733 return;
3734 }
3735
3736 if (FileReferenceManager::is_file_reference_error(status)) {
3737 string file_reference;
3738 Slice prefix = "#BASE64";
3739 Slice error_message = status.message();
3740 auto pos = error_message.rfind('#');
3741 if (pos < error_message.size() && begins_with(error_message.substr(pos), prefix)) {
3742 auto r_file_reference = base64_decode(error_message.substr(pos + prefix.size()));
3743 if (r_file_reference.is_ok()) {
3744 file_reference = r_file_reference.move_as_ok();
3745 } else {
3746 LOG(ERROR) << "Can't decode file reference from error " << status << ": " << r_file_reference.error();
3747 }
3748 } else {
3749 LOG(ERROR) << "Unexpected error, file_reference will be deleted just in case " << status;
3750 }
3751 CHECK(!node->file_ids_.empty());
3752 delete_file_reference(node->file_ids_.back(), file_reference);
3753 run_download(node, true);
3754 return;
3755 }
3756
3757 if (begins_with(status.message(), "FILE_UPLOAD_RESTART")) {
3758 if (ends_with(status.message(), "WITH_FILE_REFERENCE")) {
3759 node->upload_was_update_file_reference_ = true;
3760 }
3761 run_upload(node, {});
3762 return;
3763 }
3764 if (begins_with(status.message(), "FILE_DOWNLOAD_RESTART")) {
3765 if (ends_with(status.message(), "WITH_FILE_REFERENCE")) {
3766 node->download_was_update_file_reference_ = true;
3767 run_download(node, true);
3768 return;
3769 } else if (ends_with(status.message(), "INCREASE_PART_SIZE")) {
3770 if (try_fix_partial_local_location(node)) {
3771 run_download(node, true);
3772 return;
3773 }
3774 } else {
3775 node->can_search_locally_ = false;
3776 run_download(node, true);
3777 return;
3778 }
3779 }
3780
3781 if (!was_active) {
3782 return;
3783 }
3784
3785 // Stop everything on error
3786 do_cancel_generate(node);
3787 do_cancel_download(node);
3788 do_cancel_upload(node);
3789
3790 for (auto file_id : vector<FileId>(node->file_ids_)) {
3791 auto *info = get_file_id_info(file_id);
3792 if (info->download_priority_ != 0) {
3793 info->download_priority_ = 0;
3794 if (info->download_callback_) {
3795 info->download_callback_->on_download_error(file_id, status.clone());
3796 info->download_callback_.reset();
3797 }
3798 }
3799 if (info->upload_priority_ != 0) {
3800 info->upload_priority_ = 0;
3801 if (info->upload_callback_) {
3802 info->upload_callback_->on_upload_error(file_id, status.clone());
3803 info->upload_callback_.reset();
3804 }
3805 }
3806 }
3807 }
3808
finish_query(QueryId query_id)3809 std::pair<FileManager::Query, bool> FileManager::finish_query(QueryId query_id) {
3810 SCOPE_EXIT {
3811 queries_container_.erase(query_id);
3812 };
3813 auto query = queries_container_.get(query_id);
3814 CHECK(query != nullptr);
3815
3816 auto res = *query;
3817 auto node = get_file_node(res.file_id_);
3818 if (!node) {
3819 return std::make_pair(res, false);
3820 }
3821 bool was_active = false;
3822 if (node->generate_id_ == query_id) {
3823 node->generate_id_ = 0;
3824 node->generate_was_update_ = false;
3825 node->set_generate_priority(0, 0);
3826 was_active = true;
3827 }
3828 if (node->download_id_ == query_id) {
3829 node->download_id_ = 0;
3830 node->download_was_update_file_reference_ = false;
3831 node->is_download_started_ = false;
3832 node->set_download_priority(0);
3833 was_active = true;
3834 }
3835 if (node->upload_id_ == query_id) {
3836 node->upload_id_ = 0;
3837 node->upload_was_update_file_reference_ = false;
3838 node->set_upload_priority(0);
3839 was_active = true;
3840 }
3841 return std::make_pair(res, was_active);
3842 }
3843
get_remote(int32 key)3844 FullRemoteFileLocation *FileManager::get_remote(int32 key) {
3845 if (key == 0) {
3846 return nullptr;
3847 }
3848 return &remote_location_info_.get(key).remote_;
3849 }
3850
get_suggested_file_name(FileId file_id,const string & directory)3851 Result<string> FileManager::get_suggested_file_name(FileId file_id, const string &directory) {
3852 if (!file_id.is_valid()) {
3853 return Status::Error(400, "Invalid file identifier");
3854 }
3855 auto node = get_sync_file_node(file_id);
3856 if (!node) {
3857 return Status::Error(400, "Wrong file identifier");
3858 }
3859
3860 return ::td::get_suggested_file_name(directory, PathView(node->suggested_path()).file_name());
3861 }
3862
hangup()3863 void FileManager::hangup() {
3864 file_db_.reset();
3865 file_generate_manager_.reset();
3866 file_load_manager_.reset();
3867 while (!queries_container_.empty()) {
3868 auto ids = queries_container_.ids();
3869 for (auto id : ids) {
3870 on_error(id, Global::request_aborted_error());
3871 }
3872 }
3873 is_closed_ = true;
3874 stop();
3875 }
3876
tear_down()3877 void FileManager::tear_down() {
3878 parent_.reset();
3879 }
3880
3881 } // namespace td
3882