1 //
2 // Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2021
3 //
4 // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 //
7 #include "td/telegram/files/FileManager.h"
8 
9 #include "td/telegram/ConfigShared.h"
10 #include "td/telegram/FileReferenceManager.h"
11 #include "td/telegram/files/FileData.h"
12 #include "td/telegram/files/FileDb.h"
13 #include "td/telegram/files/FileLoaderUtils.h"
14 #include "td/telegram/files/FileLocation.h"
15 #include "td/telegram/files/FileLocation.hpp"
16 #include "td/telegram/Global.h"
17 #include "td/telegram/logevent/LogEvent.h"
18 #include "td/telegram/misc.h"
19 #include "td/telegram/SecureStorage.h"
20 #include "td/telegram/TdDb.h"
21 #include "td/telegram/TdParameters.h"
22 #include "td/telegram/Version.h"
23 
24 #include "td/actor/SleepActor.h"
25 
26 #include "td/utils/algorithm.h"
27 #include "td/utils/base64.h"
28 #include "td/utils/crypto.h"
29 #include "td/utils/filesystem.h"
30 #include "td/utils/format.h"
31 #include "td/utils/HttpUrl.h"
32 #include "td/utils/logging.h"
33 #include "td/utils/misc.h"
34 #include "td/utils/PathView.h"
35 #include "td/utils/port/FileFd.h"
36 #include "td/utils/port/path.h"
37 #include "td/utils/port/Stat.h"
38 #include "td/utils/ScopeGuard.h"
39 #include "td/utils/SliceBuilder.h"
40 #include "td/utils/StringBuilder.h"
41 #include "td/utils/Time.h"
42 #include "td/utils/tl_helpers.h"
43 #include "td/utils/tl_parsers.h"
44 
45 #include <algorithm>
46 #include <cmath>
47 #include <limits>
48 #include <numeric>
49 #include <tuple>
50 #include <unordered_set>
51 #include <utility>
52 
53 namespace td {
54 namespace {
55 constexpr int64 MAX_FILE_SIZE = 2000 * (1 << 20) /* 2000MB */;
56 }  // namespace
57 
58 int VERBOSITY_NAME(update_file) = VERBOSITY_NAME(INFO);
59 
operator <<(StringBuilder & string_builder,FileLocationSource source)60 StringBuilder &operator<<(StringBuilder &string_builder, FileLocationSource source) {
61   switch (source) {
62     case FileLocationSource::None:
63       return string_builder << "None";
64     case FileLocationSource::FromUser:
65       return string_builder << "User";
66     case FileLocationSource::FromBinlog:
67       return string_builder << "Binlog";
68     case FileLocationSource::FromDatabase:
69       return string_builder << "Database";
70     case FileLocationSource::FromServer:
71       return string_builder << "Server";
72     default:
73       UNREACHABLE();
74       return string_builder << "Unknown";
75   }
76 }
77 
operator <<(StringBuilder & string_builder,FileManager::Query::Type type)78 StringBuilder &operator<<(StringBuilder &string_builder, FileManager::Query::Type type) {
79   switch (type) {
80     case FileManager::Query::Type::UploadByHash:
81       return string_builder << "UploadByHash";
82     case FileManager::Query::Type::UploadWaitFileReference:
83       return string_builder << "UploadWaitFileReference";
84     case FileManager::Query::Type::Upload:
85       return string_builder << "Upload";
86     case FileManager::Query::Type::DownloadWaitFileReference:
87       return string_builder << "DownloadWaitFileReference";
88     case FileManager::Query::Type::DownloadReloadDialog:
89       return string_builder << "DownloadReloadDialog";
90     case FileManager::Query::Type::Download:
91       return string_builder << "Download";
92     case FileManager::Query::Type::SetContent:
93       return string_builder << "SetContent";
94     case FileManager::Query::Type::Generate:
95       return string_builder << "Generate";
96     default:
97       UNREACHABLE();
98       return string_builder << "Unknown";
99   }
100 }
101 
NewRemoteFileLocation(RemoteFileLocation remote,FileLocationSource source)102 NewRemoteFileLocation::NewRemoteFileLocation(RemoteFileLocation remote, FileLocationSource source) {
103   switch (remote.type()) {
104     case RemoteFileLocation::Type::Empty:
105       break;
106     case RemoteFileLocation::Type::Partial:
107       partial = make_unique<PartialRemoteFileLocation>(remote.partial());
108       break;
109     case RemoteFileLocation::Type::Full:
110       full = remote.full();
111       full_source = source;
112       is_full_alive = true;
113       break;
114     default:
115       UNREACHABLE();
116   }
117 }
118 
partial_or_empty() const119 RemoteFileLocation NewRemoteFileLocation::partial_or_empty() const {
120   if (partial) {
121     return RemoteFileLocation(*partial);
122   }
123   return {};
124 }
125 
operator ->() const126 FileNode *FileNodePtr::operator->() const {
127   return get();
128 }
129 
operator *() const130 FileNode &FileNodePtr::operator*() const {
131   return *get();
132 }
133 
get() const134 FileNode *FileNodePtr::get() const {
135   auto res = get_unsafe();
136   CHECK(res);
137   return res;
138 }
139 
get_remote() const140 FullRemoteFileLocation *FileNodePtr::get_remote() const {
141   return file_manager_->get_remote(file_id_.get_remote());
142 }
143 
get_unsafe() const144 FileNode *FileNodePtr::get_unsafe() const {
145   CHECK(file_manager_ != nullptr);
146   return file_manager_->get_file_node_raw(file_id_);
147 }
148 
operator bool() const149 FileNodePtr::operator bool() const {
150   return file_manager_ != nullptr && get_unsafe() != nullptr;
151 }
152 
recalc_ready_prefix_size(int64 prefix_offset,int64 ready_prefix_size)153 void FileNode::recalc_ready_prefix_size(int64 prefix_offset, int64 ready_prefix_size) {
154   if (local_.type() != LocalFileLocation::Type::Partial) {
155     return;
156   }
157   int64 new_local_ready_prefix_size;
158   if (download_offset_ == prefix_offset) {
159     new_local_ready_prefix_size = ready_prefix_size;
160   } else {
161     new_local_ready_prefix_size = Bitmask(Bitmask::Decode{}, local_.partial().ready_bitmask_)
162                                       .get_ready_prefix_size(download_offset_, local_.partial().part_size_, size_);
163   }
164   if (new_local_ready_prefix_size != local_ready_prefix_size_) {
165     VLOG(update_file) << "File " << main_file_id_ << " has changed local_ready_prefix_size from "
166                       << local_ready_prefix_size_ << " to " << new_local_ready_prefix_size;
167     local_ready_prefix_size_ = new_local_ready_prefix_size;
168     on_info_changed();
169   }
170 }
171 
init_ready_size()172 void FileNode::init_ready_size() {
173   if (local_.type() != LocalFileLocation::Type::Partial) {
174     return;
175   }
176   auto bitmask = Bitmask(Bitmask::Decode{}, local_.partial().ready_bitmask_);
177   local_ready_prefix_size_ = bitmask.get_ready_prefix_size(0, local_.partial().part_size_, size_);
178   local_ready_size_ = bitmask.get_total_size(local_.partial().part_size_, size_);
179 }
180 
set_download_offset(int64 download_offset)181 void FileNode::set_download_offset(int64 download_offset) {
182   if (download_offset < 0 || download_offset > MAX_FILE_SIZE) {
183     return;
184   }
185   if (download_offset == download_offset_) {
186     return;
187   }
188 
189   VLOG(update_file) << "File " << main_file_id_ << " has changed download_offset from " << download_offset_ << " to "
190                     << download_offset;
191   download_offset_ = download_offset;
192   is_download_offset_dirty_ = true;
193   recalc_ready_prefix_size(-1, -1);
194   on_info_changed();
195 }
set_download_limit(int64 download_limit)196 void FileNode::set_download_limit(int64 download_limit) {
197   if (download_limit < 0) {
198     return;
199   }
200   if (download_limit == download_limit_) {
201     return;
202   }
203 
204   VLOG(update_file) << "File " << main_file_id_ << " has changed download_limit from " << download_limit_ << " to "
205                     << download_limit;
206   download_limit_ = download_limit;
207   is_download_limit_dirty_ = true;
208 }
209 
drop_local_location()210 void FileNode::drop_local_location() {
211   set_local_location(LocalFileLocation(), 0, -1, -1);
212 }
213 
set_local_location(const LocalFileLocation & local,int64 ready_size,int64 prefix_offset,int64 ready_prefix_size)214 void FileNode::set_local_location(const LocalFileLocation &local, int64 ready_size, int64 prefix_offset,
215                                   int64 ready_prefix_size) {
216   if (local_ready_size_ != ready_size) {
217     VLOG(update_file) << "File " << main_file_id_ << " has changed local ready size from " << local_ready_size_
218                       << " to " << ready_size;
219     local_ready_size_ = ready_size;
220     on_info_changed();
221   }
222   if (local_ != local) {
223     VLOG(update_file) << "File " << main_file_id_ << " has changed local location";
224     local_ = local;
225 
226     recalc_ready_prefix_size(prefix_offset, ready_prefix_size);
227 
228     on_changed();
229   }
230 }
231 
set_new_remote_location(NewRemoteFileLocation new_remote)232 void FileNode::set_new_remote_location(NewRemoteFileLocation new_remote) {
233   if (new_remote.full) {
234     if (remote_.full && remote_.full.value() == new_remote.full.value()) {
235       if (remote_.full.value().get_access_hash() != new_remote.full.value().get_access_hash() ||
236           remote_.full.value().get_file_reference() != new_remote.full.value().get_file_reference() ||
237           remote_.full.value().get_source() != new_remote.full.value().get_source()) {
238         on_pmc_changed();
239       }
240     } else {
241       VLOG(update_file) << "File " << main_file_id_ << " has changed remote location";
242       on_changed();
243     }
244     remote_.full = new_remote.full;
245     remote_.full_source = new_remote.full_source;
246     remote_.is_full_alive = new_remote.is_full_alive;
247   } else {
248     if (remote_.full) {
249       VLOG(update_file) << "File " << main_file_id_ << " has lost remote location";
250       remote_.full = {};
251       remote_.is_full_alive = false;
252       remote_.full_source = FileLocationSource::None;
253       on_changed();
254     }
255   }
256 
257   if (new_remote.partial) {
258     set_partial_remote_location(*new_remote.partial, new_remote.ready_size);
259   } else {
260     delete_partial_remote_location();
261   }
262 }
delete_partial_remote_location()263 void FileNode::delete_partial_remote_location() {
264   if (remote_.partial) {
265     VLOG(update_file) << "File " << main_file_id_ << " has lost partial remote location";
266     remote_.partial.reset();
267     on_changed();
268   }
269 }
270 
set_partial_remote_location(PartialRemoteFileLocation remote,int64 ready_size)271 void FileNode::set_partial_remote_location(PartialRemoteFileLocation remote, int64 ready_size) {
272   if (remote_.is_full_alive) {
273     VLOG(update_file) << "File " << main_file_id_ << " remote is still alive, so there is NO reason to update partial";
274     return;
275   }
276   if (remote_.ready_size != ready_size) {
277     VLOG(update_file) << "File " << main_file_id_ << " has changed remote ready size from " << remote_.ready_size
278                       << " to " << ready_size;
279     remote_.ready_size = ready_size;
280     on_info_changed();
281   }
282   if (remote_.partial && *remote_.partial == remote) {
283     VLOG(update_file) << "Partial location of " << main_file_id_ << " is NOT changed";
284     return;
285   }
286   if (!remote_.partial && remote.ready_part_count_ == 0) {
287     // empty partial remote is equal to empty remote
288     VLOG(update_file) << "Partial location of " << main_file_id_
289                       << " is still empty, so there is NO reason to update it";
290     return;
291   }
292 
293   VLOG(update_file) << "File " << main_file_id_ << " partial location has changed to " << remote;
294   remote_.partial = make_unique<PartialRemoteFileLocation>(std::move(remote));
295   on_changed();
296 }
297 
delete_file_reference(Slice file_reference)298 bool FileNode::delete_file_reference(Slice file_reference) {
299   if (!remote_.full) {
300     VLOG(file_references) << "Can't delete file reference, because there is no remote location";
301     return false;
302   }
303 
304   if (!remote_.full.value().delete_file_reference(file_reference)) {
305     VLOG(file_references) << "Can't delete unmatching file reference " << format::escaped(file_reference) << ", have "
306                           << format::escaped(remote_.full.value().get_file_reference());
307     return false;
308   }
309 
310   VLOG(file_references) << "Do delete file reference of main file " << main_file_id_;
311   upload_was_update_file_reference_ = false;
312   download_was_update_file_reference_ = false;
313   on_pmc_changed();
314   return true;
315 }
316 
set_generate_location(unique_ptr<FullGenerateFileLocation> && generate)317 void FileNode::set_generate_location(unique_ptr<FullGenerateFileLocation> &&generate) {
318   bool is_changed = generate_ == nullptr ? generate != nullptr : generate == nullptr || *generate_ != *generate;
319   if (is_changed) {
320     generate_ = std::move(generate);
321     on_pmc_changed();
322   }
323 }
324 
set_size(int64 size)325 void FileNode::set_size(int64 size) {
326   if (size_ != size) {
327     VLOG(update_file) << "File " << main_file_id_ << " has changed size to " << size;
328     size_ = size;
329     on_changed();
330   }
331 }
332 
set_expected_size(int64 expected_size)333 void FileNode::set_expected_size(int64 expected_size) {
334   if (expected_size_ != expected_size) {
335     VLOG(update_file) << "File " << main_file_id_ << " has changed expected size to " << expected_size;
336     expected_size_ = expected_size;
337     on_changed();
338   }
339 }
340 
set_remote_name(string remote_name)341 void FileNode::set_remote_name(string remote_name) {
342   if (remote_name_ != remote_name) {
343     remote_name_ = std::move(remote_name);
344     on_pmc_changed();
345   }
346 }
347 
set_url(string url)348 void FileNode::set_url(string url) {
349   if (url_ != url) {
350     VLOG(update_file) << "File " << main_file_id_ << " has changed URL to " << url;
351     url_ = std::move(url);
352     on_changed();
353   }
354 }
355 
set_owner_dialog_id(DialogId owner_id)356 void FileNode::set_owner_dialog_id(DialogId owner_id) {
357   if (owner_dialog_id_ != owner_id) {
358     owner_dialog_id_ = owner_id;
359     on_pmc_changed();
360   }
361 }
362 
set_encryption_key(FileEncryptionKey key)363 void FileNode::set_encryption_key(FileEncryptionKey key) {
364   if (encryption_key_ != key) {
365     encryption_key_ = std::move(key);
366     on_pmc_changed();
367   }
368 }
369 
set_upload_pause(FileId upload_pause)370 void FileNode::set_upload_pause(FileId upload_pause) {
371   if (upload_pause_ != upload_pause) {
372     LOG(INFO) << "Change file " << main_file_id_ << " upload_pause from " << upload_pause_ << " to " << upload_pause;
373     upload_pause_ = upload_pause;
374   }
375 }
376 
set_download_priority(int8 priority)377 void FileNode::set_download_priority(int8 priority) {
378   if ((download_priority_ == 0) != (priority == 0)) {
379     VLOG(update_file) << "File " << main_file_id_ << " has changed download priority to " << priority;
380     on_info_changed();
381   }
382   download_priority_ = priority;
383 }
384 
set_upload_priority(int8 priority)385 void FileNode::set_upload_priority(int8 priority) {
386   if (!remote_.is_full_alive && (upload_priority_ == 0) != (priority == 0)) {
387     VLOG(update_file) << "File " << main_file_id_ << " has changed upload priority to " << priority;
388     on_info_changed();
389   }
390   upload_priority_ = priority;
391 }
392 
set_generate_priority(int8 download_priority,int8 upload_priority)393 void FileNode::set_generate_priority(int8 download_priority, int8 upload_priority) {
394   if ((generate_download_priority_ == 0) != (download_priority == 0) ||
395       (generate_upload_priority_ == 0) != (upload_priority == 0)) {
396     VLOG(update_file) << "File " << main_file_id_ << " has changed generate priority to " << download_priority << "/"
397                       << upload_priority;
398     on_info_changed();
399   }
400   generate_priority_ = max(download_priority, upload_priority);
401   generate_download_priority_ = download_priority;
402   generate_upload_priority_ = upload_priority;
403 }
404 
on_changed()405 void FileNode::on_changed() {
406   on_pmc_changed();
407   on_info_changed();
408 }
409 
on_info_changed()410 void FileNode::on_info_changed() {
411   info_changed_flag_ = true;
412 }
413 
on_pmc_changed()414 void FileNode::on_pmc_changed() {
415   pmc_changed_flag_ = true;
416 }
417 
need_info_flush() const418 bool FileNode::need_info_flush() const {
419   return info_changed_flag_;
420 }
421 
need_pmc_flush() const422 bool FileNode::need_pmc_flush() const {
423   if (!pmc_changed_flag_) {
424     return false;
425   }
426 
427   // already in pmc
428   if (pmc_id_.is_valid()) {
429     return true;
430   }
431 
432   // We must save encryption key
433   if (!encryption_key_.empty()) {
434     // && remote_.type() != RemoteFileLocation::Type::Empty
435     return true;
436   }
437 
438   bool has_generate_location = generate_ != nullptr;
439   // Do not save "#file_id#" conversion.
440   if (has_generate_location && begins_with(generate_->conversion_, "#file_id#")) {
441     has_generate_location = false;
442   }
443 
444   if (remote_.full/* &&
445       (has_generate_location || local_.type() != LocalFileLocation::Type::Empty)*/) {
446     // we need to always save file sources
447     return true;
448   }
449   if (local_.type() == LocalFileLocation::Type::Full && (has_generate_location || remote_.full || remote_.partial)) {
450     return true;
451   }
452 
453   // TODO: Generate location with constant conversion
454 
455   return false;
456 }
457 
on_pmc_flushed()458 void FileNode::on_pmc_flushed() {
459   pmc_changed_flag_ = false;
460 }
on_info_flushed()461 void FileNode::on_info_flushed() {
462   info_changed_flag_ = false;
463 }
464 
suggested_path() const465 string FileNode::suggested_path() const {
466   if (!remote_name_.empty()) {
467     return remote_name_;
468   }
469   if (!url_.empty()) {
470     auto file_name = get_url_file_name(url_);
471     if (!file_name.empty()) {
472       return file_name;
473     }
474   }
475   if (generate_ != nullptr) {
476     if (!generate_->original_path_.empty()) {
477       return generate_->original_path_;
478     }
479   }
480   return local_.file_name().str();
481 }
482 
483 /*** FileView ***/
has_local_location() const484 bool FileView::has_local_location() const {
485   return node_->local_.type() == LocalFileLocation::Type::Full;
486 }
487 
local_location() const488 const FullLocalFileLocation &FileView::local_location() const {
489   CHECK(has_local_location());
490   return node_->local_.full();
491 }
492 
has_remote_location() const493 bool FileView::has_remote_location() const {
494   return static_cast<bool>(node_->remote_.full);
495 }
496 
has_alive_remote_location() const497 bool FileView::has_alive_remote_location() const {
498   return node_->remote_.is_full_alive;
499 }
500 
has_active_upload_remote_location() const501 bool FileView::has_active_upload_remote_location() const {
502   if (!has_remote_location()) {
503     return false;
504   }
505   if (!has_alive_remote_location()) {
506     return false;
507   }
508   if (main_remote_location().is_encrypted_any()) {
509     return true;
510   }
511   return main_remote_location().has_file_reference();
512 }
513 
has_active_download_remote_location() const514 bool FileView::has_active_download_remote_location() const {
515   if (!has_remote_location()) {
516     return false;
517   }
518   if (remote_location().is_encrypted_any()) {
519     return true;
520   }
521   return remote_location().has_file_reference();
522 }
523 
remote_location() const524 const FullRemoteFileLocation &FileView::remote_location() const {
525   CHECK(has_remote_location());
526   auto *remote = node_.get_remote();
527   if (remote) {
528     return *remote;
529   }
530   return node_->remote_.full.value();
531 }
532 
main_remote_location() const533 const FullRemoteFileLocation &FileView::main_remote_location() const {
534   CHECK(has_remote_location());
535   return node_->remote_.full.value();
536 }
537 
has_generate_location() const538 bool FileView::has_generate_location() const {
539   return node_->generate_ != nullptr;
540 }
541 
generate_location() const542 const FullGenerateFileLocation &FileView::generate_location() const {
543   CHECK(has_generate_location());
544   return *node_->generate_;
545 }
546 
size() const547 int64 FileView::size() const {
548   return node_->size_;
549 }
550 
get_allocated_local_size() const551 int64 FileView::get_allocated_local_size() const {
552   auto file_path = path();
553   if (file_path.empty()) {
554     return 0;
555   }
556   auto r_stat = stat(file_path);
557   if (r_stat.is_error()) {
558     return 0;
559   }
560   return r_stat.ok().real_size_;
561 }
562 
expected_size(bool may_guess) const563 int64 FileView::expected_size(bool may_guess) const {
564   if (node_->size_ != 0) {
565     return node_->size_;
566   }
567   int64 current_size = local_total_size();  // TODO: this is not the best approximation
568   if (node_->expected_size_ != 0) {
569     return max(current_size, node_->expected_size_);
570   }
571   if (may_guess && node_->local_.type() == LocalFileLocation::Type::Partial) {
572     current_size *= 3;
573   }
574   return current_size;
575 }
576 
is_downloading() const577 bool FileView::is_downloading() const {
578   return node_->download_priority_ != 0 || node_->generate_download_priority_ != 0;
579 }
580 
download_offset() const581 int64 FileView::download_offset() const {
582   return node_->download_offset_;
583 }
584 
downloaded_prefix(int64 offset) const585 int64 FileView::downloaded_prefix(int64 offset) const {
586   switch (node_->local_.type()) {
587     case LocalFileLocation::Type::Empty:
588       return 0;
589     case LocalFileLocation::Type::Full:
590       if (offset < node_->size_) {
591         return node_->size_ - offset;
592       }
593       return 0;
594     case LocalFileLocation::Type::Partial:
595       if (is_encrypted_secure()) {
596         // File is not decrypted and verified yet
597         return 0;
598       }
599       return Bitmask(Bitmask::Decode{}, node_->local_.partial().ready_bitmask_)
600           .get_ready_prefix_size(offset, node_->local_.partial().part_size_, node_->size_);
601     default:
602       UNREACHABLE();
603       return 0;
604   }
605 }
606 
local_prefix_size() const607 int64 FileView::local_prefix_size() const {
608   switch (node_->local_.type()) {
609     case LocalFileLocation::Type::Full:
610       return node_->download_offset_ <= node_->size_ ? node_->size_ - node_->download_offset_ : 0;
611     case LocalFileLocation::Type::Partial: {
612       if (is_encrypted_secure()) {
613         // File is not decrypted and verified yet
614         return 0;
615       }
616       return node_->local_ready_prefix_size_;
617     }
618     default:
619       return 0;
620   }
621 }
local_total_size() const622 int64 FileView::local_total_size() const {
623   switch (node_->local_.type()) {
624     case LocalFileLocation::Type::Empty:
625       return 0;
626     case LocalFileLocation::Type::Full:
627       return node_->size_;
628     case LocalFileLocation::Type::Partial:
629       VLOG(update_file) << "Have local_ready_prefix_size = " << node_->local_ready_prefix_size_
630                         << " and local_ready_size = " << node_->local_ready_size_;
631       return max(node_->local_ready_prefix_size_, node_->local_ready_size_);
632     default:
633       UNREACHABLE();
634       return 0;
635   }
636 }
637 
is_uploading() const638 bool FileView::is_uploading() const {
639   return node_->upload_priority_ != 0 || node_->generate_upload_priority_ != 0;
640 }
641 
remote_size() const642 int64 FileView::remote_size() const {
643   if (node_->remote_.is_full_alive) {
644     return node_->size_;
645   }
646   if (node_->remote_.partial) {
647     auto part_size = static_cast<int64>(node_->remote_.partial->part_size_);
648     auto ready_part_count = node_->remote_.partial->ready_part_count_;
649     auto remote_ready_size = node_->remote_.ready_size;
650     VLOG(update_file) << "Have part_size = " << part_size << ", remote_ready_part_count = " << ready_part_count
651                       << ", remote_ready_size = " << remote_ready_size << ", size = " << size();
652     auto res = max(part_size * ready_part_count, remote_ready_size);
653     if (size() != 0 && size() < res) {
654       res = size();
655     }
656     return res;
657   }
658   return node_->remote_.ready_size;  //???
659 }
660 
path() const661 string FileView::path() const {
662   switch (node_->local_.type()) {
663     case LocalFileLocation::Type::Full:
664       return node_->local_.full().path_;
665     case LocalFileLocation::Type::Partial:
666       return node_->local_.partial().path_;
667     default:
668       return "";
669   }
670 }
671 
has_url() const672 bool FileView::has_url() const {
673   return !node_->url_.empty();
674 }
675 
url() const676 const string &FileView::url() const {
677   return node_->url_;
678 }
679 
remote_name() const680 const string &FileView::remote_name() const {
681   return node_->remote_name_;
682 }
683 
suggested_path() const684 string FileView::suggested_path() const {
685   return node_->suggested_path();
686 }
687 
owner_dialog_id() const688 DialogId FileView::owner_dialog_id() const {
689   return node_->owner_dialog_id_;
690 }
691 
get_by_hash() const692 bool FileView::get_by_hash() const {
693   return node_->get_by_hash_;
694 }
695 
FileView(ConstFileNodePtr node)696 FileView::FileView(ConstFileNodePtr node) : node_(node) {
697 }
698 
empty() const699 bool FileView::empty() const {
700   return !node_;
701 }
702 
can_download_from_server() const703 bool FileView::can_download_from_server() const {
704   if (!has_remote_location()) {
705     return false;
706   }
707   if (remote_location().file_type_ == FileType::Encrypted && encryption_key().empty()) {
708     return false;
709   }
710   if (remote_location().is_web()) {
711     return true;
712   }
713   if (remote_location().get_dc_id().is_empty()) {
714     return false;
715   }
716   if (!remote_location().is_encrypted_any() && !remote_location().has_file_reference() &&
717       ((node_->download_id_ == 0 && node_->download_was_update_file_reference_) || !node_->remote_.is_full_alive)) {
718     return false;
719   }
720   return true;
721 }
722 
can_generate() const723 bool FileView::can_generate() const {
724   return has_generate_location();
725 }
726 
can_delete() const727 bool FileView::can_delete() const {
728   if (has_local_location()) {
729     return begins_with(local_location().path_, get_files_dir(get_type()));
730   }
731   return node_->local_.type() == LocalFileLocation::Type::Partial;
732 }
733 
get_unique_id(const FullGenerateFileLocation & location)734 string FileView::get_unique_id(const FullGenerateFileLocation &location) {
735   return base64url_encode(zero_encode('\xff' + serialize(location)));
736 }
737 
get_unique_id(const FullRemoteFileLocation & location)738 string FileView::get_unique_id(const FullRemoteFileLocation &location) {
739   return base64url_encode(zero_encode(serialize(location.as_unique())));
740 }
741 
get_persistent_id(const FullGenerateFileLocation & location)742 string FileView::get_persistent_id(const FullGenerateFileLocation &location) {
743   auto binary = serialize(location);
744 
745   binary = zero_encode(binary);
746   binary.push_back(FileNode::PERSISTENT_ID_VERSION_MAP);
747   return base64url_encode(binary);
748 }
749 
get_persistent_id(const FullRemoteFileLocation & location)750 string FileView::get_persistent_id(const FullRemoteFileLocation &location) {
751   auto binary = serialize(location);
752 
753   binary = zero_encode(binary);
754   binary.push_back(static_cast<char>(narrow_cast<uint8>(Version::Next) - 1));
755   binary.push_back(FileNode::PERSISTENT_ID_VERSION);
756   return base64url_encode(binary);
757 }
758 
get_persistent_file_id() const759 string FileView::get_persistent_file_id() const {
760   if (!empty()) {
761     if (has_alive_remote_location()) {
762       return get_persistent_id(remote_location());
763     } else if (has_url()) {
764       return url();
765     } else if (has_generate_location() && begins_with(generate_location().conversion_, "#map#")) {
766       return get_persistent_id(generate_location());
767     }
768   }
769   return string();
770 }
771 
get_unique_file_id() const772 string FileView::get_unique_file_id() const {
773   if (!empty()) {
774     if (has_alive_remote_location()) {
775       if (!remote_location().is_web()) {
776         return get_unique_id(remote_location());
777       }
778     } else if (has_generate_location() && begins_with(generate_location().conversion_, "#map#")) {
779       return get_unique_id(generate_location());
780     }
781   }
782   return string();
783 }
784 
785 /*** FileManager ***/
786 static int merge_choose_remote_location(const FullRemoteFileLocation &x, FileLocationSource x_source,
787                                         const FullRemoteFileLocation &y, FileLocationSource y_source);
788 
789 namespace {
prepare_path_for_pmc(FileType file_type,string & path)790 void prepare_path_for_pmc(FileType file_type, string &path) {
791   path = PathView::relative(path, get_files_base_dir(file_type)).str();
792 }
793 }  // namespace
794 
FileManager(unique_ptr<Context> context)795 FileManager::FileManager(unique_ptr<Context> context) : context_(std::move(context)) {
796   if (G()->parameters().use_file_db) {
797     file_db_ = G()->td_db()->get_file_db_shared();
798   }
799 
800   parent_ = context_->create_reference();
801   next_file_id();
802   next_file_node_id();
803 
804   std::unordered_set<string> dir_paths;
805   for (int32 i = 0; i < MAX_FILE_TYPE; i++) {
806     dir_paths.insert(get_files_dir(static_cast<FileType>(i)));
807   }
808   // add both temp dirs
809   dir_paths.insert(get_files_temp_dir(FileType::Encrypted));
810   dir_paths.insert(get_files_temp_dir(FileType::Video));
811 
812   for (const auto &path : dir_paths) {
813     auto status = mkdir(path, 0750);
814     if (status.is_error()) {
815       auto r_stat = stat(path);
816       if (r_stat.is_ok() && r_stat.ok().is_dir_) {
817         LOG(ERROR) << "Creation of directory \"" << path << "\" failed with " << status << ", but directory exists";
818       } else {
819         LOG(ERROR) << "Creation of directory \"" << path << "\" failed with " << status;
820       }
821     }
822 #if TD_ANDROID
823     FileFd::open(path + ".nomedia", FileFd::Create | FileFd::Read).ignore();
824 #endif
825   };
826 
827   G()->td_db()->with_db_path([bad_paths = &bad_paths_](CSlice path) { bad_paths->insert(path.str()); });
828 }
829 
init_actor()830 void FileManager::init_actor() {
831   file_load_manager_ = create_actor_on_scheduler<FileLoadManager>("FileLoadManager", G()->get_slow_net_scheduler_id(),
832                                                                   actor_shared(this), context_->create_reference());
833   file_generate_manager_ = create_actor_on_scheduler<FileGenerateManager>(
834       "FileGenerateManager", G()->get_slow_net_scheduler_id(), context_->create_reference());
835 }
836 
~FileManager()837 FileManager::~FileManager() {
838 }
839 
fix_file_extension(Slice file_name,Slice file_type,Slice file_extension)840 string FileManager::fix_file_extension(Slice file_name, Slice file_type, Slice file_extension) {
841   return (file_name.empty() ? file_type : file_name).str() + "." + file_extension.str();
842 }
843 
get_file_name(FileType file_type,Slice path)844 string FileManager::get_file_name(FileType file_type, Slice path) {
845   PathView path_view(path);
846   auto file_name = path_view.file_name();
847   auto extension = path_view.extension();
848   switch (file_type) {
849     case FileType::Thumbnail:
850       if (extension != "jpg" && extension != "jpeg" && extension != "webp") {
851         return fix_file_extension(file_name, "thumbnail", "jpg");
852       }
853       break;
854     case FileType::ProfilePhoto:
855     case FileType::Photo:
856       if (extension != "jpg" && extension != "jpeg" && extension != "gif" && extension != "png" && extension != "tif" &&
857           extension != "bmp") {
858         return fix_file_extension(file_name, "photo", "jpg");
859       }
860       break;
861     case FileType::VoiceNote:
862       if (extension != "ogg" && extension != "oga" && extension != "mp3" && extension != "mpeg3" &&
863           extension != "m4a" && extension != "opus") {
864         return fix_file_extension(file_name, "voice", "oga");
865       }
866       break;
867     case FileType::Video:
868     case FileType::VideoNote:
869       if (extension != "mov" && extension != "3gp" && extension != "mpeg4" && extension != "mp4") {
870         return fix_file_extension(file_name, "video", "mp4");
871       }
872       break;
873     case FileType::Audio:
874       if (extension != "ogg" && extension != "oga" && extension != "mp3" && extension != "mpeg3" &&
875           extension != "m4a") {
876         return fix_file_extension(file_name, "audio", "mp3");
877       }
878       break;
879     case FileType::Wallpaper:
880     case FileType::Background:
881       if (extension != "jpg" && extension != "jpeg" && extension != "png") {
882         return fix_file_extension(file_name, "wallpaper", "jpg");
883       }
884       break;
885     case FileType::Sticker:
886       if (extension != "webp" && extension != "tgs") {
887         return fix_file_extension(file_name, "sticker", "webp");
888       }
889       break;
890     case FileType::Document:
891     case FileType::Animation:
892     case FileType::Encrypted:
893     case FileType::Temp:
894     case FileType::EncryptedThumbnail:
895     case FileType::Secure:
896     case FileType::SecureRaw:
897     case FileType::DocumentAsFile:
898       break;
899     default:
900       UNREACHABLE();
901   }
902   return file_name.str();
903 }
904 
are_modification_times_equal(int64 old_mtime,int64 new_mtime)905 bool FileManager::are_modification_times_equal(int64 old_mtime, int64 new_mtime) {
906   if (old_mtime == new_mtime) {
907     return true;
908   }
909   if (old_mtime < new_mtime) {
910     return false;
911   }
912   if (old_mtime - new_mtime == 1000000000 && old_mtime % 1000000000 == 0 && new_mtime % 2000000000 == 0) {
913     // FAT32 has 2 seconds mtime resolution, but file system sometimes reports odd modification time
914     return true;
915   }
916   return false;
917 }
918 
check_local_location(FullLocalFileLocation & location,int64 & size,bool skip_file_size_checks)919 Status FileManager::check_local_location(FullLocalFileLocation &location, int64 &size, bool skip_file_size_checks) {
920   constexpr int64 MAX_THUMBNAIL_SIZE = 200 * (1 << 10) - 1 /* 200 KB - 1 B */;
921   constexpr int64 MAX_PHOTO_SIZE = 10 * (1 << 20) /* 10 MB */;
922   constexpr int64 DEFAULT_VIDEO_NOTE_SIZE_MAX = 12 * (1 << 20) /* 12 MB */;
923 
924   if (location.path_.empty()) {
925     return Status::Error(400, "File must have non-empty path");
926   }
927   auto r_path = realpath(location.path_, true);
928   if (r_path.is_error()) {
929     return Status::Error(400, "Can't find real file path");
930   }
931   location.path_ = r_path.move_as_ok();
932   if (bad_paths_.count(location.path_) != 0) {
933     return Status::Error(400, "Sending of internal database files is forbidden");
934   }
935   auto r_stat = stat(location.path_);
936   if (r_stat.is_error()) {
937     return Status::Error(400, "Can't get stat about the file");
938   }
939   auto stat = r_stat.move_as_ok();
940   if (!stat.is_reg_) {
941     return Status::Error(400, "File must be a regular file");
942   }
943   if (stat.size_ < 0) {
944     // TODO is it possible?
945     return Status::Error(400, "File is too big");
946   }
947   if (stat.size_ == 0) {
948     return Status::Error(400, "File must be non-empty");
949   }
950 
951   if (size == 0) {
952     size = stat.size_;
953   }
954   if (location.mtime_nsec_ == 0) {
955     VLOG(file_loader) << "Set file \"" << location.path_ << "\" modification time to " << stat.mtime_nsec_;
956     location.mtime_nsec_ = stat.mtime_nsec_;
957   } else if (!are_modification_times_equal(location.mtime_nsec_, stat.mtime_nsec_)) {
958     VLOG(file_loader) << "File \"" << location.path_ << "\" was modified: old mtime = " << location.mtime_nsec_
959                       << ", new mtime = " << stat.mtime_nsec_;
960     return Status::Error(400, PSLICE() << "File \"" << location.path_ << "\" was modified");
961   }
962   if (skip_file_size_checks) {
963     return Status::OK();
964   }
965   if ((location.file_type_ == FileType::Thumbnail || location.file_type_ == FileType::EncryptedThumbnail) &&
966       size > MAX_THUMBNAIL_SIZE && !begins_with(PathView(location.path_).file_name(), "map")) {
967     return Status::Error(400, PSLICE() << "File \"" << location.path_ << "\" is too big for a thumbnail "
968                                        << tag("size", format::as_size(size)));
969   }
970   if (size > MAX_FILE_SIZE) {
971     return Status::Error(400, PSLICE() << "File \"" << location.path_ << "\" of size " << size << " bytes is too big");
972   }
973   if (location.file_type_ == FileType::Photo && size > MAX_PHOTO_SIZE) {
974     return Status::Error(
975         400, PSLICE() << "File \"" << location.path_ << "\" of size " << size << " bytes is too big for a photo");
976   }
977   if (location.file_type_ == FileType::VideoNote &&
978       size > G()->shared_config().get_option_integer("video_note_size_max", DEFAULT_VIDEO_NOTE_SIZE_MAX)) {
979     return Status::Error(
980         400, PSLICE() << "File \"" << location.path_ << "\" of size " << size << " bytes is too big for a video note");
981   }
982   return Status::OK();
983 }
984 
check_partial_local_location(const PartialLocalFileLocation & location)985 static Status check_partial_local_location(const PartialLocalFileLocation &location) {
986   TRY_RESULT(stat, stat(location.path_));
987   if (!stat.is_reg_) {
988     if (stat.is_dir_) {
989       return Status::Error(PSLICE() << "Can't use directory \"" << location.path_ << "\" as a file path");
990     }
991     return Status::Error("File must be a regular file");
992   }
993   // can't check mtime. Hope nobody will mess with this file in our temporary dir.
994   return Status::OK();
995 }
996 
check_local_location(FileNodePtr node)997 Status FileManager::check_local_location(FileNodePtr node) {
998   Status status;
999   if (node->local_.type() == LocalFileLocation::Type::Full) {
1000     status = check_local_location(node->local_.full(), node->size_, false);
1001   } else if (node->local_.type() == LocalFileLocation::Type::Partial) {
1002     status = check_partial_local_location(node->local_.partial());
1003   }
1004   if (status.is_error()) {
1005     node->drop_local_location();
1006     try_flush_node(node, "check_local_location");
1007   }
1008   return status;
1009 }
1010 
try_fix_partial_local_location(FileNodePtr node)1011 bool FileManager::try_fix_partial_local_location(FileNodePtr node) {
1012   LOG(INFO) << "Trying to fix partial local location";
1013   if (node->local_.type() != LocalFileLocation::Type::Partial) {
1014     LOG(INFO) << "   failed - not a partial location";
1015     return false;
1016   }
1017   auto partial = node->local_.partial();
1018   if (!partial.iv_.empty()) {
1019     // can't recalc iv_
1020     LOG(INFO) << "   failed - partial location has nonempty iv";
1021     return false;
1022   }
1023   if (partial.part_size_ >= 512 * (1 << 10)) {
1024     LOG(INFO) << "   failed - too big part_size already: " << partial.part_size_;
1025     return false;
1026   }
1027   auto old_part_size = partial.part_size_;
1028   int new_part_size = 512 * (1 << 10);
1029   auto k = new_part_size / old_part_size;
1030   Bitmask mask(Bitmask::Decode(), partial.ready_bitmask_);
1031   auto new_mask = mask.compress(k);
1032 
1033   partial.part_size_ = new_part_size;
1034   partial.ready_bitmask_ = new_mask.encode();
1035 
1036   auto ready_size = new_mask.get_total_size(partial.part_size_, node->size_);
1037   node->set_local_location(LocalFileLocation(std::move(partial)), ready_size, -1, -1);
1038   LOG(INFO) << "   ok: increase part_size " << old_part_size << "->" << new_part_size;
1039   return true;
1040 }
1041 
get_file_id_info(FileId file_id)1042 FileManager::FileIdInfo *FileManager::get_file_id_info(FileId file_id) {
1043   CHECK(static_cast<size_t>(file_id.get()) < file_id_info_.size());
1044   return &file_id_info_[file_id.get()];
1045 }
1046 
dup_file_id(FileId file_id)1047 FileId FileManager::dup_file_id(FileId file_id) {
1048   int32 file_node_id;
1049   auto *file_node = get_file_node_raw(file_id, &file_node_id);
1050   if (!file_node) {
1051     return FileId();
1052   }
1053   auto result = FileId(create_file_id(file_node_id, file_node).get(), file_id.get_remote());
1054   LOG(INFO) << "Dup file " << file_id << " to " << result;
1055   return result;
1056 }
1057 
create_file_id(int32 file_node_id,FileNode * file_node)1058 FileId FileManager::create_file_id(int32 file_node_id, FileNode *file_node) {
1059   auto file_id = next_file_id();
1060   get_file_id_info(file_id)->node_id_ = file_node_id;
1061   file_node->file_ids_.push_back(file_id);
1062   return file_id;
1063 }
1064 
try_forget_file_id(FileId file_id)1065 void FileManager::try_forget_file_id(FileId file_id) {
1066   auto *info = get_file_id_info(file_id);
1067   if (info->send_updates_flag_ || info->pin_flag_ || info->sent_file_id_flag_) {
1068     return;
1069   }
1070   auto file_node = get_file_node(file_id);
1071   if (file_node->main_file_id_ == file_id) {
1072     return;
1073   }
1074 
1075   LOG(DEBUG) << "Forget file " << file_id;
1076   bool is_removed = td::remove(file_node->file_ids_, file_id);
1077   CHECK(is_removed);
1078   *info = FileIdInfo();
1079   empty_file_ids_.push_back(file_id.get());
1080 }
1081 
register_empty(FileType type)1082 FileId FileManager::register_empty(FileType type) {
1083   return register_local(FullLocalFileLocation(type, "", 0), DialogId(), 0, false, true).ok();
1084 }
1085 
on_file_unlink(const FullLocalFileLocation & location)1086 void FileManager::on_file_unlink(const FullLocalFileLocation &location) {
1087   // TODO: remove file from the database too
1088   auto it = local_location_to_file_id_.find(location);
1089   if (it == local_location_to_file_id_.end()) {
1090     return;
1091   }
1092   auto file_id = it->second;
1093   auto file_node = get_sync_file_node(file_id);
1094   CHECK(file_node);
1095   file_node->drop_local_location();
1096   try_flush_node_info(file_node, "on_file_unlink");
1097 }
1098 
register_local(FullLocalFileLocation location,DialogId owner_dialog_id,int64 size,bool get_by_hash,bool force,bool skip_file_size_checks)1099 Result<FileId> FileManager::register_local(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size,
1100                                            bool get_by_hash, bool force, bool skip_file_size_checks) {
1101   // TODO: use get_by_hash
1102   FileData data;
1103   data.local_ = LocalFileLocation(std::move(location));
1104   data.owner_dialog_id_ = owner_dialog_id;
1105   data.size_ = size;
1106   return register_file(std::move(data), FileLocationSource::None /*won't be used*/, "register_local", force,
1107                        skip_file_size_checks);
1108 }
1109 
register_remote(FullRemoteFileLocation location,FileLocationSource file_location_source,DialogId owner_dialog_id,int64 size,int64 expected_size,string remote_name)1110 FileId FileManager::register_remote(FullRemoteFileLocation location, FileLocationSource file_location_source,
1111                                     DialogId owner_dialog_id, int64 size, int64 expected_size, string remote_name) {
1112   FileData data;
1113   auto url = location.get_url();
1114   data.remote_ = RemoteFileLocation(std::move(location));
1115   data.owner_dialog_id_ = owner_dialog_id;
1116   data.size_ = size;
1117   data.expected_size_ = expected_size;
1118   data.remote_name_ = std::move(remote_name);
1119 
1120   auto file_id = register_file(std::move(data), file_location_source, "register_remote", false).move_as_ok();
1121   if (!url.empty()) {
1122     auto file_node = get_file_node(file_id);
1123     CHECK(file_node);
1124     file_node->set_url(url);
1125   }
1126   return file_id;
1127 }
1128 
register_url(string url,FileType file_type,FileLocationSource file_location_source,DialogId owner_dialog_id)1129 FileId FileManager::register_url(string url, FileType file_type, FileLocationSource file_location_source,
1130                                  DialogId owner_dialog_id) {
1131   auto file_id = register_generate(file_type, file_location_source, url, "#url#", owner_dialog_id, 0).ok();
1132   auto file_node = get_file_node(file_id);
1133   CHECK(file_node);
1134   file_node->set_url(url);
1135   return file_id;
1136 }
1137 
register_generate(FileType file_type,FileLocationSource file_location_source,string original_path,string conversion,DialogId owner_dialog_id,int64 expected_size)1138 Result<FileId> FileManager::register_generate(FileType file_type, FileLocationSource file_location_source,
1139                                               string original_path, string conversion, DialogId owner_dialog_id,
1140                                               int64 expected_size) {
1141   // add #mtime# into conversion
1142   if (!original_path.empty() && conversion[0] != '#' && PathView(original_path).is_absolute()) {
1143     auto file_paths = log_interface->get_file_paths();
1144     if (!td::contains(file_paths, original_path)) {
1145       auto r_stat = stat(original_path);
1146       uint64 mtime = r_stat.is_ok() ? r_stat.ok().mtime_nsec_ : 0;
1147       conversion = PSTRING() << "#mtime#" << lpad0(to_string(mtime), 20) << '#' << conversion;
1148     }
1149   }
1150 
1151   FileData data;
1152   data.generate_ =
1153       td::make_unique<FullGenerateFileLocation>(file_type, std::move(original_path), std::move(conversion));
1154   data.owner_dialog_id_ = owner_dialog_id;
1155   data.expected_size_ = expected_size;
1156   return register_file(std::move(data), file_location_source, "register_generate", false);
1157 }
1158 
register_file(FileData && data,FileLocationSource file_location_source,const char * source,bool force,bool skip_file_size_checks)1159 Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource file_location_source, const char *source,
1160                                           bool force, bool skip_file_size_checks) {
1161   bool has_remote = data.remote_.type() == RemoteFileLocation::Type::Full;
1162   bool has_generate = data.generate_ != nullptr;
1163   if (data.local_.type() == LocalFileLocation::Type::Full && !force) {
1164     if (file_location_source == FileLocationSource::FromBinlog ||
1165         file_location_source == FileLocationSource::FromDatabase) {
1166       PathView path_view(data.local_.full().path_);
1167       if (path_view.is_relative()) {
1168         data.local_.full().path_ = PSTRING()
1169                                    << get_files_base_dir(data.local_.full().file_type_) << data.local_.full().path_;
1170       }
1171     }
1172 
1173     auto status = check_local_location(data.local_.full(), data.size_, skip_file_size_checks);
1174     if (status.is_error()) {
1175       LOG(INFO) << "Invalid " << data.local_.full() << ": " << status << " from " << source;
1176       data.local_ = LocalFileLocation();
1177       if (data.remote_.type() == RemoteFileLocation::Type::Partial) {
1178         data.remote_ = {};
1179       }
1180 
1181       if (!has_remote && !has_generate) {
1182         return std::move(status);
1183       }
1184     }
1185   }
1186   bool has_local = data.local_.type() == LocalFileLocation::Type::Full;
1187   bool has_location = has_local || has_remote || has_generate;
1188   if (!has_location) {
1189     return Status::Error(400, "No location");
1190   }
1191 
1192   FileId file_id = next_file_id();
1193 
1194   LOG(INFO) << "Register file data " << data << " as " << file_id << " from " << source;
1195   // create FileNode
1196   auto file_node_id = next_file_node_id();
1197   auto &node = file_nodes_[file_node_id];
1198   node = td::make_unique<FileNode>(std::move(data.local_), NewRemoteFileLocation(data.remote_, file_location_source),
1199                                    std::move(data.generate_), data.size_, data.expected_size_,
1200                                    std::move(data.remote_name_), std::move(data.url_), data.owner_dialog_id_,
1201                                    std::move(data.encryption_key_), file_id, static_cast<int8>(has_remote));
1202   node->pmc_id_ = FileDbId(data.pmc_id_);
1203   get_file_id_info(file_id)->node_id_ = file_node_id;
1204   node->file_ids_.push_back(file_id);
1205 
1206   FileView file_view(get_file_node(file_id));
1207 
1208   std::vector<FileId> to_merge;
1209   auto register_location = [&](const auto &location, auto &mp) {
1210     auto &other_id = mp[location];
1211     if (other_id.empty()) {
1212       other_id = file_id;
1213       get_file_id_info(file_id)->pin_flag_ = true;
1214       return true;
1215     } else {
1216       to_merge.push_back(other_id);
1217       return false;
1218     }
1219   };
1220   bool new_remote = false;
1221   int32 remote_key = 0;
1222   if (file_view.has_remote_location()) {
1223     RemoteInfo info{file_view.remote_location(), file_location_source, file_id};
1224     remote_key = remote_location_info_.add(info);
1225     auto &stored_info = remote_location_info_.get(remote_key);
1226     if (stored_info.file_id_ == file_id) {
1227       get_file_id_info(file_id)->pin_flag_ = true;
1228       new_remote = true;
1229     } else {
1230       to_merge.push_back(stored_info.file_id_);
1231       if (merge_choose_remote_location(file_view.remote_location(), file_location_source, stored_info.remote_,
1232                                        stored_info.file_location_source_) == 0) {
1233         stored_info.remote_ = file_view.remote_location();
1234         stored_info.file_location_source_ = file_location_source;
1235       }
1236     }
1237   }
1238   bool new_local = false;
1239   if (file_view.has_local_location()) {
1240     new_local = register_location(file_view.local_location(), local_location_to_file_id_);
1241   }
1242   bool new_generate = false;
1243   if (file_view.has_generate_location()) {
1244     new_generate = register_location(file_view.generate_location(), generate_location_to_file_id_);
1245   }
1246   td::unique(to_merge);
1247 
1248   int new_cnt = new_remote + new_local + new_generate;
1249   if (data.pmc_id_ == 0 && file_db_ && new_cnt > 0) {
1250     node->need_load_from_pmc_ = true;
1251   }
1252   bool no_sync_merge = to_merge.size() == 1 && new_cnt == 0;
1253   for (auto id : to_merge) {
1254     // may invalidate node
1255     merge(file_id, id, no_sync_merge).ignore();
1256   }
1257 
1258   try_flush_node(get_file_node(file_id), "register_file");
1259   auto main_file_id = get_file_node(file_id)->main_file_id_;
1260   try_forget_file_id(file_id);
1261   for (auto file_source_id : data.file_source_ids_) {
1262     VLOG(file_references) << "Loaded " << data.file_source_ids_ << " for file " << main_file_id << " from " << source;
1263     if (file_source_id.is_valid()) {
1264       context_->add_file_source(main_file_id, file_source_id);
1265     }
1266   }
1267   return FileId(main_file_id.get(), remote_key);
1268 }
1269 
1270 // 0 -- choose x
1271 // 1 -- choose y
1272 // 2 -- choose any
merge_choose_local_location(const LocalFileLocation & x,const LocalFileLocation & y)1273 static int merge_choose_local_location(const LocalFileLocation &x, const LocalFileLocation &y) {
1274   auto x_type = static_cast<int32>(x.type());
1275   auto y_type = static_cast<int32>(y.type());
1276   if (x_type != y_type) {
1277     return x_type < y_type;
1278   }
1279   return 2;
1280 }
1281 
merge_choose_file_source_location(FileLocationSource x,FileLocationSource y)1282 static int merge_choose_file_source_location(FileLocationSource x, FileLocationSource y) {
1283   return static_cast<int>(x) < static_cast<int>(y);
1284 }
1285 
merge_choose_remote_location(const FullRemoteFileLocation & x,FileLocationSource x_source,const FullRemoteFileLocation & y,FileLocationSource y_source)1286 static int merge_choose_remote_location(const FullRemoteFileLocation &x, FileLocationSource x_source,
1287                                         const FullRemoteFileLocation &y, FileLocationSource y_source) {
1288   LOG(INFO) << "Choose between " << x << " from " << x_source << " and " << y << " from " << y_source;
1289   if (x.is_web() != y.is_web()) {
1290     return x.is_web();  // prefer non-web
1291   }
1292   auto x_ref = x.has_file_reference();
1293   auto y_ref = y.has_file_reference();
1294   if (x_ref || y_ref) {
1295     if (x_ref != y_ref) {
1296       return !x_ref;
1297     }
1298     if (x.get_file_reference() != y.get_file_reference()) {
1299       return merge_choose_file_source_location(x_source, y_source);
1300     }
1301   }
1302   if ((x.get_access_hash() != y.get_access_hash() || x.get_source() != y.get_source()) &&
1303       (x_source != y_source || x.is_web() || x.get_id() == y.get_id())) {
1304     return merge_choose_file_source_location(x_source, y_source);
1305   }
1306   return 2;
1307 }
1308 
merge_choose_remote_location(const NewRemoteFileLocation & x,const NewRemoteFileLocation & y)1309 static int merge_choose_remote_location(const NewRemoteFileLocation &x, const NewRemoteFileLocation &y) {
1310   if (x.is_full_alive != y.is_full_alive) {
1311     return !x.is_full_alive;
1312   }
1313   if (x.is_full_alive) {
1314     return merge_choose_remote_location(x.full.value(), x.full_source, y.full.value(), y.full_source);
1315   }
1316   if (!x.partial != !y.partial) {
1317     return !x.partial;
1318   }
1319   return 2;
1320 }
1321 
merge_choose_generate_location(const unique_ptr<FullGenerateFileLocation> & x,const unique_ptr<FullGenerateFileLocation> & y)1322 static int merge_choose_generate_location(const unique_ptr<FullGenerateFileLocation> &x,
1323                                           const unique_ptr<FullGenerateFileLocation> &y) {
1324   int x_empty = (x == nullptr);
1325   int y_empty = (y == nullptr);
1326   if (x_empty != y_empty) {
1327     return x_empty ? 1 : 0;
1328   }
1329   if (!x_empty && *x != *y) {
1330     bool x_has_mtime = begins_with(x->conversion_, "#mtime#");
1331     bool y_has_mtime = begins_with(y->conversion_, "#mtime#");
1332     if (x_has_mtime != y_has_mtime) {
1333       return x_has_mtime ? 0 : 1;
1334     }
1335     return x->conversion_ >= y->conversion_
1336                ? 0
1337                : 1;  // the bigger conversion, the bigger mtime or at least more stable choise
1338   }
1339   return 2;
1340 }
1341 
1342 // -1 -- error
merge_choose_size(int64 x,int64 y)1343 static int merge_choose_size(int64 x, int64 y) {
1344   if (x == 0) {
1345     return 1;
1346   }
1347   if (y == 0) {
1348     return 0;
1349   }
1350   if (x != y) {
1351     return -1;
1352   }
1353   return 2;
1354 }
1355 
merge_choose_expected_size(int64 x,int64 y)1356 static int merge_choose_expected_size(int64 x, int64 y) {
1357   if (x == 0) {
1358     return 1;
1359   }
1360   if (y == 0) {
1361     return 0;
1362   }
1363   return 2;
1364 }
1365 
merge_choose_name(Slice x,Slice y)1366 static int merge_choose_name(Slice x, Slice y) {
1367   if (x.empty() != y.empty()) {
1368     return x.empty() > y.empty();
1369   }
1370   return 2;
1371 }
1372 
merge_choose_owner(DialogId x,DialogId y)1373 static int merge_choose_owner(DialogId x, DialogId y) {
1374   if (x.is_valid() != y.is_valid()) {
1375     return x.is_valid() < y.is_valid();
1376   }
1377   return 2;
1378 }
1379 
merge_choose_main_file_id(FileId a,int8 a_priority,FileId b,int8 b_priority)1380 static int merge_choose_main_file_id(FileId a, int8 a_priority, FileId b, int8 b_priority) {
1381   if (a_priority != b_priority) {
1382     return a_priority < b_priority;
1383   }
1384   return a.get() > b.get();
1385 }
1386 
merge_choose_encryption_key(const FileEncryptionKey & a,const FileEncryptionKey & b)1387 static int merge_choose_encryption_key(const FileEncryptionKey &a, const FileEncryptionKey &b) {
1388   if (a.empty() != b.empty()) {
1389     return a.empty() > b.empty();
1390   }
1391   if (a != b) {
1392     return -1;
1393   }
1394   return 2;
1395 }
1396 
do_cancel_download(FileNodePtr node)1397 void FileManager::do_cancel_download(FileNodePtr node) {
1398   if (node->download_id_ == 0) {
1399     return;
1400   }
1401   send_closure(file_load_manager_, &FileLoadManager::cancel, node->download_id_);
1402   node->download_id_ = 0;
1403   node->is_download_started_ = false;
1404   node->download_was_update_file_reference_ = false;
1405   node->set_download_priority(0);
1406 }
1407 
do_cancel_upload(FileNodePtr node)1408 void FileManager::do_cancel_upload(FileNodePtr node) {
1409   if (node->upload_id_ == 0) {
1410     return;
1411   }
1412   send_closure(file_load_manager_, &FileLoadManager::cancel, node->upload_id_);
1413   node->upload_id_ = 0;
1414   node->upload_was_update_file_reference_ = false;
1415   node->set_upload_priority(0);
1416 }
1417 
do_cancel_generate(FileNodePtr node)1418 void FileManager::do_cancel_generate(FileNodePtr node) {
1419   if (node->generate_id_ == 0) {
1420     return;
1421   }
1422   send_closure(file_generate_manager_, &FileGenerateManager::cancel, node->generate_id_);
1423   node->generate_id_ = 0;
1424   node->generate_was_update_ = false;
1425   node->set_generate_priority(0, 0);
1426 }
1427 
merge(FileId x_file_id,FileId y_file_id,bool no_sync)1428 Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) {
1429   LOG(DEBUG) << "Merge new file " << x_file_id << " and old file " << y_file_id;
1430 
1431   if (!x_file_id.is_valid()) {
1432     return Status::Error("First file_id is invalid");
1433   }
1434   FileNodePtr x_node = no_sync ? get_file_node(x_file_id) : get_sync_file_node(x_file_id);
1435   if (!x_node) {
1436     return Status::Error(PSLICE() << "Can't merge files. First identifier is invalid: " << x_file_id << " and "
1437                                   << y_file_id);
1438   }
1439 
1440   if (!y_file_id.is_valid()) {
1441     LOG(DEBUG) << "Old file is invalid";
1442     return x_node->main_file_id_;
1443   }
1444   FileNodePtr y_node = get_file_node(y_file_id);
1445   if (!y_node) {
1446     return Status::Error(PSLICE() << "Can't merge files. Second identifier is invalid: " << x_file_id << " and "
1447                                   << y_file_id);
1448   }
1449 
1450   if (x_file_id == x_node->upload_pause_) {
1451     x_node->set_upload_pause(FileId());
1452   }
1453   if (x_node.get() == y_node.get()) {
1454     LOG(DEBUG) << "Files are already merged";
1455     return x_node->main_file_id_;
1456   }
1457   if (y_file_id == y_node->upload_pause_) {
1458     y_node->set_upload_pause(FileId());
1459   }
1460 
1461   if (x_node->remote_.full && y_node->remote_.full && !x_node->remote_.full.value().is_web() &&
1462       !y_node->remote_.full.value().is_web() && y_node->remote_.is_full_alive &&
1463       x_node->remote_.full_source == FileLocationSource::FromServer &&
1464       y_node->remote_.full_source == FileLocationSource::FromServer &&
1465       x_node->remote_.full.value().get_dc_id() != y_node->remote_.full.value().get_dc_id()) {
1466     LOG(ERROR) << "File remote location was changed from " << y_node->remote_.full.value() << " to "
1467                << x_node->remote_.full.value();
1468   }
1469 
1470   bool drop_last_successful_force_reupload_time = x_node->last_successful_force_reupload_time_ <= 0 &&
1471                                                   x_node->remote_.full &&
1472                                                   x_node->remote_.full_source == FileLocationSource::FromServer;
1473 
1474   auto count_local = [](auto &node) {
1475     return std::accumulate(node->file_ids_.begin(), node->file_ids_.end(), 0,
1476                            [](const auto &x, const auto &y) { return x + (y.get_remote() != 0); });
1477   };
1478   auto x_local_file_ids = count_local(x_node);
1479   auto y_local_file_ids = count_local(y_node);
1480   if (x_local_file_ids + y_local_file_ids > 100) {
1481   }
1482 
1483   if (y_node->file_ids_.size() >= 100 || x_node->file_ids_.size() >= 100) {
1484     LOG(INFO) << "Merge files with " << x_local_file_ids << '/' << x_node->file_ids_.size() << " and "
1485               << y_local_file_ids << '/' << y_node->file_ids_.size() << " file IDs";
1486   }
1487 
1488   FileNodePtr nodes[] = {x_node, y_node, x_node};
1489   FileNodeId node_ids[] = {get_file_id_info(x_file_id)->node_id_, get_file_id_info(y_file_id)->node_id_};
1490   int trusted_by_source = merge_choose_file_source_location(x_node->remote_.full_source, y_node->remote_.full_source);
1491 
1492   int local_i = merge_choose_local_location(x_node->local_, y_node->local_);
1493   int remote_i = merge_choose_remote_location(x_node->remote_, y_node->remote_);
1494   int generate_i = merge_choose_generate_location(x_node->generate_, y_node->generate_);
1495   int size_i = merge_choose_size(x_node->size_, y_node->size_);
1496   int expected_size_i = merge_choose_expected_size(x_node->expected_size_, y_node->expected_size_);
1497   int remote_name_i = merge_choose_name(x_node->remote_name_, y_node->remote_name_);
1498   int url_i = merge_choose_name(x_node->url_, y_node->url_);
1499   int owner_i = merge_choose_owner(x_node->owner_dialog_id_, y_node->owner_dialog_id_);
1500   int encryption_key_i = merge_choose_encryption_key(x_node->encryption_key_, y_node->encryption_key_);
1501   int main_file_id_i = merge_choose_main_file_id(x_node->main_file_id_, x_node->main_file_id_priority_,
1502                                                  y_node->main_file_id_, y_node->main_file_id_priority_);
1503 
1504   if (size_i == -1) {
1505     return Status::Error(PSLICE() << "Can't merge files. Different size: " << x_node->size_ << " and "
1506                                   << y_node->size_);
1507   }
1508   if (encryption_key_i == -1) {
1509     if (nodes[remote_i]->remote_.full && nodes[local_i]->local_.type() != LocalFileLocation::Type::Partial) {
1510       LOG(ERROR) << "Different encryption key in files, but lets choose same key as remote location";
1511       encryption_key_i = remote_i;
1512     } else {
1513       return Status::Error("Can't merge files. Different encryption keys");
1514     }
1515   }
1516 
1517   // prefer more trusted source
1518   if (remote_name_i == 2) {
1519     remote_name_i = trusted_by_source;
1520   }
1521   if (url_i == 2) {
1522     url_i = trusted_by_source;
1523   }
1524   if (expected_size_i == 2) {
1525     expected_size_i = trusted_by_source;
1526   }
1527 
1528   int node_i =
1529       std::make_tuple(y_node->pmc_id_.is_valid(), x_node->pmc_id_, y_node->file_ids_.size(), main_file_id_i == 1) >
1530       std::make_tuple(x_node->pmc_id_.is_valid(), y_node->pmc_id_, x_node->file_ids_.size(), main_file_id_i == 0);
1531 
1532   auto other_node_i = 1 - node_i;
1533   FileNodePtr node = nodes[node_i];
1534   FileNodePtr other_node = nodes[other_node_i];
1535   auto file_view = FileView(node);
1536 
1537   LOG(DEBUG) << "Have x_node->pmc_id_ = " << x_node->pmc_id_.get() << ", y_node->pmc_id_ = " << y_node->pmc_id_.get()
1538              << ", x_node_size = " << x_node->file_ids_.size() << ", y_node_size = " << y_node->file_ids_.size()
1539              << ", node_i = " << node_i << ", local_i = " << local_i << ", remote_i = " << remote_i
1540              << ", generate_i = " << generate_i << ", size_i = " << size_i << ", remote_name_i = " << remote_name_i
1541              << ", url_i = " << url_i << ", owner_i = " << owner_i << ", encryption_key_i = " << encryption_key_i
1542              << ", main_file_id_i = " << main_file_id_i << ", trusted_by_source = " << trusted_by_source
1543              << ", x_source = " << x_node->remote_.full_source << ", y_source = " << y_node->remote_.full_source;
1544   if (local_i == other_node_i) {
1545     do_cancel_download(node);
1546     node->set_download_offset(other_node->download_offset_);
1547     node->set_local_location(other_node->local_, other_node->local_ready_size_, other_node->download_offset_,
1548                              other_node->local_ready_prefix_size_);
1549     node->download_id_ = other_node->download_id_;
1550     node->download_was_update_file_reference_ = other_node->download_was_update_file_reference_;
1551     node->is_download_started_ |= other_node->is_download_started_;
1552     node->set_download_priority(other_node->download_priority_);
1553     other_node->download_id_ = 0;
1554     other_node->download_was_update_file_reference_ = false;
1555     other_node->is_download_started_ = false;
1556     other_node->download_priority_ = 0;
1557     other_node->download_offset_ = 0;
1558     other_node->local_ready_prefix_size_ = 0;
1559 
1560     //do_cancel_generate(node);
1561     //node->set_generate_location(std::move(other_node->generate_));
1562     //node->generate_id_ = other_node->generate_id_;
1563     //node->set_generate_priority(other_node->generate_download_priority_, other_node->generate_upload_priority_);
1564     //other_node->generate_id_ = 0;
1565     //other_node->generate_was_update_ = false;
1566     //other_node->generate_priority_ = 0;
1567     //other_node->generate_download_priority_ = 0;
1568     //other_node->generate_upload_priority_ = 0;
1569   } else {
1570     do_cancel_download(other_node);
1571     //do_cancel_generate(other_node);
1572   }
1573 
1574   if (remote_i == other_node_i) {
1575     do_cancel_upload(node);
1576     node->set_new_remote_location(std::move(other_node->remote_));
1577     node->upload_id_ = other_node->upload_id_;
1578     node->upload_was_update_file_reference_ = other_node->upload_was_update_file_reference_;
1579     node->set_upload_priority(other_node->upload_priority_);
1580     node->set_upload_pause(other_node->upload_pause_);
1581     other_node->upload_id_ = 0;
1582     other_node->upload_was_update_file_reference_ = false;
1583     other_node->upload_priority_ = 0;
1584     other_node->set_upload_pause(FileId());
1585   } else {
1586     do_cancel_upload(other_node);
1587   }
1588 
1589   if (generate_i == other_node_i) {
1590     do_cancel_generate(node);
1591     node->set_generate_location(std::move(other_node->generate_));
1592     node->generate_id_ = other_node->generate_id_;
1593     node->set_generate_priority(other_node->generate_download_priority_, other_node->generate_upload_priority_);
1594     other_node->generate_id_ = 0;
1595     other_node->generate_priority_ = 0;
1596     other_node->generate_download_priority_ = 0;
1597     other_node->generate_upload_priority_ = 0;
1598   } else {
1599     do_cancel_generate(other_node);
1600   }
1601 
1602   if (size_i == other_node_i) {
1603     node->set_size(other_node->size_);
1604   }
1605 
1606   if (expected_size_i == other_node_i) {
1607     node->set_expected_size(other_node->expected_size_);
1608   }
1609 
1610   if (remote_name_i == other_node_i) {
1611     node->set_remote_name(other_node->remote_name_);
1612   }
1613 
1614   if (url_i == other_node_i) {
1615     node->set_url(other_node->url_);
1616   }
1617 
1618   if (owner_i == other_node_i) {
1619     node->set_owner_dialog_id(other_node->owner_dialog_id_);
1620   }
1621 
1622   if (encryption_key_i == other_node_i) {
1623     node->set_encryption_key(other_node->encryption_key_);
1624     nodes[node_i]->set_encryption_key(nodes[encryption_key_i]->encryption_key_);
1625   }
1626   node->need_load_from_pmc_ |= other_node->need_load_from_pmc_;
1627   node->can_search_locally_ &= other_node->can_search_locally_;
1628   node->upload_prefer_small_ |= other_node->upload_prefer_small_;
1629 
1630   if (drop_last_successful_force_reupload_time) {
1631     node->last_successful_force_reupload_time_ = -1e10;
1632   } else if (other_node->last_successful_force_reupload_time_ > node->last_successful_force_reupload_time_) {
1633     node->last_successful_force_reupload_time_ = other_node->last_successful_force_reupload_time_;
1634   }
1635 
1636   if (main_file_id_i == other_node_i) {
1637     context_->on_merge_files(other_node->main_file_id_, node->main_file_id_);
1638     node->main_file_id_ = other_node->main_file_id_;
1639     node->main_file_id_priority_ = other_node->main_file_id_priority_;
1640   } else {
1641     context_->on_merge_files(node->main_file_id_, other_node->main_file_id_);
1642   }
1643 
1644   bool send_updates_flag = false;
1645   auto other_pmc_id = other_node->pmc_id_;
1646   node->file_ids_.insert(node->file_ids_.end(), other_node->file_ids_.begin(), other_node->file_ids_.end());
1647 
1648   for (auto file_id : other_node->file_ids_) {
1649     auto file_id_info = get_file_id_info(file_id);
1650     CHECK(file_id_info->node_id_ == node_ids[other_node_i]);
1651     file_id_info->node_id_ = node_ids[node_i];
1652     send_updates_flag |= file_id_info->send_updates_flag_;
1653   }
1654   other_node = {};
1655 
1656   if (send_updates_flag) {
1657     // node might not changed, but other_node might changed, so we need to send update anyway
1658     VLOG(update_file) << "File " << node->main_file_id_ << " has been merged";
1659     node->on_info_changed();
1660   }
1661 
1662   // Check if some download/upload queries are ready
1663   for (auto file_id : vector<FileId>(node->file_ids_)) {
1664     auto *info = get_file_id_info(file_id);
1665     if (info->download_priority_ != 0 && file_view.has_local_location()) {
1666       info->download_priority_ = 0;
1667       if (info->download_callback_) {
1668         info->download_callback_->on_download_ok(file_id);
1669         info->download_callback_.reset();
1670       }
1671     }
1672     if (info->upload_priority_ != 0 && file_view.has_active_upload_remote_location()) {
1673       info->upload_priority_ = 0;
1674       if (info->upload_callback_) {
1675         info->upload_callback_->on_upload_ok(file_id, nullptr);
1676         info->upload_callback_.reset();
1677       }
1678     }
1679   }
1680 
1681   file_nodes_[node_ids[other_node_i]] = nullptr;
1682 
1683   run_generate(node);
1684   run_download(node, false);
1685   run_upload(node, {});
1686 
1687   if (other_pmc_id.is_valid()) {
1688     // node might not changed, but we need to merge nodes in pmc anyway
1689     node->on_pmc_changed();
1690   }
1691   try_flush_node_full(node, node_i != remote_i, node_i != local_i, node_i != generate_i, other_pmc_id);
1692 
1693   return node->main_file_id_;
1694 }
1695 
add_file_source(FileId file_id,FileSourceId file_source_id)1696 void FileManager::add_file_source(FileId file_id, FileSourceId file_source_id) {
1697   auto node = get_file_node(file_id);
1698   if (!node) {
1699     return;
1700   }
1701 
1702   CHECK(file_source_id.is_valid());
1703   if (context_->add_file_source(node->main_file_id_, file_source_id)) {
1704     node->on_pmc_changed();
1705     try_flush_node_pmc(node, "add_file_source");
1706   }
1707 }
1708 
remove_file_source(FileId file_id,FileSourceId file_source_id)1709 void FileManager::remove_file_source(FileId file_id, FileSourceId file_source_id) {
1710   auto node = get_file_node(file_id);
1711   if (!node) {
1712     return;
1713   }
1714 
1715   CHECK(file_source_id.is_valid());
1716   if (context_->remove_file_source(node->main_file_id_, file_source_id)) {
1717     node->on_pmc_changed();
1718     try_flush_node_pmc(node, "remove_file_source");
1719   }
1720 }
1721 
change_files_source(FileSourceId file_source_id,const vector<FileId> & old_file_ids,const vector<FileId> & new_file_ids)1722 void FileManager::change_files_source(FileSourceId file_source_id, const vector<FileId> &old_file_ids,
1723                                       const vector<FileId> &new_file_ids) {
1724   if (old_file_ids == new_file_ids) {
1725     return;
1726   }
1727   CHECK(file_source_id.is_valid());
1728 
1729   auto old_main_file_ids = get_main_file_ids(old_file_ids);
1730   auto new_main_file_ids = get_main_file_ids(new_file_ids);
1731   for (auto file_id : old_main_file_ids) {
1732     auto it = new_main_file_ids.find(file_id);
1733     if (it == new_main_file_ids.end()) {
1734       remove_file_source(file_id, file_source_id);
1735     } else {
1736       new_main_file_ids.erase(it);
1737     }
1738   }
1739   for (auto file_id : new_main_file_ids) {
1740     add_file_source(file_id, file_source_id);
1741   }
1742 }
1743 
on_file_reference_repaired(FileId file_id,FileSourceId file_source_id,Result<Unit> && result,Promise<Unit> && promise)1744 void FileManager::on_file_reference_repaired(FileId file_id, FileSourceId file_source_id, Result<Unit> &&result,
1745                                              Promise<Unit> &&promise) {
1746   TRY_STATUS_PROMISE(promise, G()->close_status());
1747 
1748   auto file_view = get_file_view(file_id);
1749   CHECK(!file_view.empty());
1750   if (result.is_ok() &&
1751       (!file_view.has_active_upload_remote_location() || !file_view.has_active_download_remote_location())) {
1752     result = Status::Error("No active remote location");
1753   }
1754   if (result.is_error() && result.error().code() != 429 && result.error().code() < 500) {
1755     VLOG(file_references) << "Invalid " << file_source_id << " " << result.error();
1756     remove_file_source(file_id, file_source_id);
1757   }
1758   promise.set_result(std::move(result));
1759 }
1760 
get_main_file_ids(const vector<FileId> & file_ids)1761 std::unordered_set<FileId, FileIdHash> FileManager::get_main_file_ids(const vector<FileId> &file_ids) {
1762   std::unordered_set<FileId, FileIdHash> result;
1763   for (auto file_id : file_ids) {
1764     auto node = get_file_node(file_id);
1765     if (node) {
1766       result.insert(node->main_file_id_);
1767     }
1768   }
1769   return result;
1770 }
1771 
try_flush_node_full(FileNodePtr node,bool new_remote,bool new_local,bool new_generate,FileDbId other_pmc_id)1772 void FileManager::try_flush_node_full(FileNodePtr node, bool new_remote, bool new_local, bool new_generate,
1773                                       FileDbId other_pmc_id) {
1774   if (node->need_pmc_flush()) {
1775     if (file_db_) {
1776       load_from_pmc(node, true, true, true);
1777       flush_to_pmc(node, new_remote, new_local, new_generate, "try_flush_node_full");
1778       if (other_pmc_id.is_valid() && node->pmc_id_ != other_pmc_id) {
1779         file_db_->set_file_data_ref(other_pmc_id, node->pmc_id_);
1780       }
1781     }
1782     node->on_pmc_flushed();
1783   }
1784 
1785   try_flush_node_info(node, "try_flush_node_full");
1786 }
1787 
try_flush_node(FileNodePtr node,const char * source)1788 void FileManager::try_flush_node(FileNodePtr node, const char *source) {
1789   try_flush_node_pmc(node, source);
1790   try_flush_node_info(node, source);
1791 }
1792 
try_flush_node_pmc(FileNodePtr node,const char * source)1793 void FileManager::try_flush_node_pmc(FileNodePtr node, const char *source) {
1794   if (node->need_pmc_flush()) {
1795     if (file_db_) {
1796       load_from_pmc(node, true, true, true);
1797       flush_to_pmc(node, false, false, false, source);
1798     }
1799     node->on_pmc_flushed();
1800   }
1801 }
1802 
try_flush_node_info(FileNodePtr node,const char * source)1803 void FileManager::try_flush_node_info(FileNodePtr node, const char *source) {
1804   if (node->need_info_flush()) {
1805     for (auto file_id : vector<FileId>(node->file_ids_)) {
1806       auto *info = get_file_id_info(file_id);
1807       if (info->send_updates_flag_) {
1808         VLOG(update_file) << "Send UpdateFile about file " << file_id << " from " << source;
1809         context_->on_file_updated(file_id);
1810       }
1811     }
1812     node->on_info_flushed();
1813   }
1814 }
1815 
clear_from_pmc(FileNodePtr node)1816 void FileManager::clear_from_pmc(FileNodePtr node) {
1817   if (!file_db_) {
1818     return;
1819   }
1820   if (node->pmc_id_.empty()) {
1821     return;
1822   }
1823 
1824   LOG(INFO) << "Delete files " << format::as_array(node->file_ids_) << " from pmc";
1825   FileData data;
1826   auto file_view = FileView(node);
1827   if (file_view.has_local_location()) {
1828     data.local_ = node->local_;
1829   }
1830   if (file_view.has_remote_location()) {
1831     data.remote_ = RemoteFileLocation(*node->remote_.full);
1832   }
1833   if (file_view.has_generate_location()) {
1834     data.generate_ = make_unique<FullGenerateFileLocation>(*node->generate_);
1835   }
1836   file_db_->clear_file_data(node->pmc_id_, data);
1837   node->pmc_id_ = FileDbId();
1838 }
1839 
flush_to_pmc(FileNodePtr node,bool new_remote,bool new_local,bool new_generate,const char * source)1840 void FileManager::flush_to_pmc(FileNodePtr node, bool new_remote, bool new_local, bool new_generate,
1841                                const char *source) {
1842   if (!file_db_) {
1843     return;
1844   }
1845   FileView view(node);
1846   bool create_flag = false;
1847   if (node->pmc_id_.empty()) {
1848     create_flag = true;
1849     node->pmc_id_ = file_db_->create_pmc_id();
1850   }
1851 
1852   FileData data;
1853   data.pmc_id_ = node->pmc_id_.get();
1854   data.local_ = node->local_;
1855   if (data.local_.type() == LocalFileLocation::Type::Full) {
1856     prepare_path_for_pmc(data.local_.full().file_type_, data.local_.full().path_);
1857   }
1858   if (node->remote_.full) {
1859     data.remote_ = RemoteFileLocation(node->remote_.full.value());
1860   } else if (node->remote_.partial) {
1861     data.remote_ = RemoteFileLocation(*node->remote_.partial);
1862   }
1863   if (node->generate_ != nullptr && !begins_with(node->generate_->conversion_, "#file_id#")) {
1864     data.generate_ = make_unique<FullGenerateFileLocation>(*node->generate_);
1865   }
1866 
1867   // TODO: not needed when GenerateLocation has constant conversion
1868   if (data.remote_.type() != RemoteFileLocation::Type::Full && data.local_.type() != LocalFileLocation::Type::Full) {
1869     data.local_ = LocalFileLocation();
1870     data.remote_ = RemoteFileLocation();
1871   }
1872   if (data.remote_.type() != RemoteFileLocation::Type::Full && node->encryption_key_.is_secure()) {
1873     data.remote_ = RemoteFileLocation();
1874   }
1875 
1876   data.size_ = node->size_;
1877   data.expected_size_ = node->expected_size_;
1878   data.remote_name_ = node->remote_name_;
1879   data.encryption_key_ = node->encryption_key_;
1880   data.url_ = node->url_;
1881   data.owner_dialog_id_ = node->owner_dialog_id_;
1882   data.file_source_ids_ = context_->get_some_file_sources(view.file_id());
1883   VLOG(file_references) << "Save file " << view.file_id() << " to database with " << data.file_source_ids_ << " from "
1884                         << source;
1885 
1886   file_db_->set_file_data(node->pmc_id_, data, (create_flag || new_remote), (create_flag || new_local),
1887                           (create_flag || new_generate));
1888 }
1889 
get_file_node_raw(FileId file_id,FileNodeId * file_node_id)1890 FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_id) {
1891   if (file_id.get() <= 0 || file_id.get() >= static_cast<int32>(file_id_info_.size())) {
1892     return nullptr;
1893   }
1894   FileNodeId node_id = file_id_info_[file_id.get()].node_id_;
1895   if (node_id == 0) {
1896     return nullptr;
1897   }
1898   if (file_node_id != nullptr) {
1899     *file_node_id = node_id;
1900   }
1901   return file_nodes_[node_id].get();
1902 }
1903 
get_sync_file_node(FileId file_id)1904 FileNodePtr FileManager::get_sync_file_node(FileId file_id) {
1905   auto file_node = get_file_node(file_id);
1906   if (!file_node) {
1907     return {};
1908   }
1909   load_from_pmc(file_node, true, true, true);
1910   return file_node;
1911 }
1912 
load_from_pmc(FileNodePtr node,bool new_remote,bool new_local,bool new_generate)1913 void FileManager::load_from_pmc(FileNodePtr node, bool new_remote, bool new_local, bool new_generate) {
1914   if (!node->need_load_from_pmc_) {
1915     return;
1916   }
1917   auto file_id = node->main_file_id_;
1918   node->need_load_from_pmc_ = false;
1919   if (!file_db_) {
1920     return;
1921   }
1922   auto file_view = get_file_view(file_id);
1923   CHECK(!file_view.empty());
1924 
1925   FullRemoteFileLocation remote;
1926   FullLocalFileLocation local;
1927   FullGenerateFileLocation generate;
1928   new_remote &= file_view.has_remote_location();
1929   if (new_remote) {
1930     remote = file_view.remote_location();
1931   }
1932   new_local &= file_view.has_local_location();
1933   if (new_local) {
1934     local = file_view.local_location();
1935     prepare_path_for_pmc(local.file_type_, local.path_);
1936   }
1937   new_generate &= file_view.has_generate_location();
1938   if (new_generate) {
1939     generate = file_view.generate_location();
1940   }
1941 
1942   LOG(DEBUG) << "Load from pmc " << file_id << "/" << file_view.file_id() << ", new_remote = " << new_remote
1943              << ", new_local = " << new_local << ", new_generate = " << new_generate;
1944   auto load = [&](auto location) {
1945     TRY_RESULT(file_data, file_db_->get_file_data_sync(location));
1946     TRY_RESULT(new_file_id,
1947                register_file(std::move(file_data), FileLocationSource::FromDatabase, "load_from_pmc", false));
1948     TRY_RESULT(main_file_id, merge(file_id, new_file_id));
1949     file_id = main_file_id;
1950     return Status::OK();
1951   };
1952   if (new_remote) {
1953     load(remote).ignore();
1954   }
1955   if (new_local) {
1956     load(local).ignore();
1957   }
1958   if (new_generate) {
1959     load(generate).ignore();
1960   }
1961 }
1962 
set_encryption_key(FileId file_id,FileEncryptionKey key)1963 bool FileManager::set_encryption_key(FileId file_id, FileEncryptionKey key) {
1964   auto node = get_sync_file_node(file_id);
1965   if (!node) {
1966     return false;
1967   }
1968   auto view = FileView(node);
1969   if (view.has_local_location() && view.has_remote_location()) {
1970     return false;
1971   }
1972   if (!node->encryption_key_.empty()) {
1973     return false;
1974   }
1975   node->set_encryption_key(std::move(key));
1976   try_flush_node_pmc(node, "set_encryption_key");
1977   return true;
1978 }
1979 
set_content(FileId file_id,BufferSlice bytes)1980 bool FileManager::set_content(FileId file_id, BufferSlice bytes) {
1981   if (G()->shared_config().get_option_boolean("ignore_inline_thumbnails")) {
1982     return false;
1983   }
1984 
1985   auto node = get_sync_file_node(file_id);
1986   if (!node) {
1987     return false;
1988   }
1989 
1990   if (node->local_.type() == LocalFileLocation::Type::Full) {
1991     // There was no download so we don't need an update
1992     return true;
1993   }
1994 
1995   if (node->download_priority_ == FROM_BYTES_PRIORITY) {
1996     return true;
1997   }
1998 
1999   do_cancel_download(node);
2000 
2001   auto *file_info = get_file_id_info(file_id);
2002   file_info->download_priority_ = FROM_BYTES_PRIORITY;
2003 
2004   node->set_download_priority(FROM_BYTES_PRIORITY);
2005 
2006   QueryId id = queries_container_.create(Query{file_id, Query::Type::SetContent});
2007   node->download_id_ = id;
2008   node->is_download_started_ = true;
2009   send_closure(file_load_manager_, &FileLoadManager::from_bytes, id, node->remote_.full.value().file_type_,
2010                std::move(bytes), node->suggested_path());
2011   return true;
2012 }
2013 
get_content(FileId file_id,Promise<BufferSlice> promise)2014 void FileManager::get_content(FileId file_id, Promise<BufferSlice> promise) {
2015   auto node = get_sync_file_node(file_id);
2016   if (!node) {
2017     return promise.set_error(Status::Error("Unknown file_id"));
2018   }
2019   auto status = check_local_location(node);
2020   status.ignore();
2021 
2022   auto file_view = FileView(node);
2023   if (!file_view.has_local_location()) {
2024     return promise.set_error(Status::Error("No local location"));
2025   }
2026 
2027   send_closure(file_load_manager_, &FileLoadManager::get_content, node->local_.full(), std::move(promise));
2028 }
2029 
read_file_part(FileId file_id,int32 offset,int32 count,int left_tries,Promise<td_api::object_ptr<td_api::filePart>> promise)2030 void FileManager::read_file_part(FileId file_id, int32 offset, int32 count, int left_tries,
2031                                  Promise<td_api::object_ptr<td_api::filePart>> promise) {
2032   TRY_STATUS_PROMISE(promise, G()->close_status());
2033 
2034   if (!file_id.is_valid()) {
2035     return promise.set_error(Status::Error(400, "File identifier is invalid"));
2036   }
2037   auto node = get_sync_file_node(file_id);
2038   if (!node) {
2039     return promise.set_error(Status::Error(400, "File not found"));
2040   }
2041   if (offset < 0) {
2042     return promise.set_error(Status::Error(400, "Parameter offset must be non-negative"));
2043   }
2044   if (count < 0) {
2045     return promise.set_error(Status::Error(400, "Parameter count must be non-negative"));
2046   }
2047 
2048   auto file_view = FileView(node);
2049 
2050   if (count == 0) {
2051     count = narrow_cast<int32>(file_view.downloaded_prefix(offset));
2052     if (count == 0) {
2053       return promise.set_value(td_api::make_object<td_api::filePart>());
2054     }
2055   } else if (file_view.downloaded_prefix(offset) < static_cast<int64>(count)) {
2056     // TODO this check is safer to do in another thread
2057     return promise.set_error(Status::Error(400, "There is not enough downloaded bytes in the file to read"));
2058   }
2059 
2060   const string *path = nullptr;
2061   bool is_partial = false;
2062   if (file_view.has_local_location()) {
2063     path = &file_view.local_location().path_;
2064     if (!begins_with(*path, get_files_dir(file_view.get_type()))) {
2065       return promise.set_error(Status::Error(400, "File is not inside the cache"));
2066     }
2067   } else {
2068     CHECK(node->local_.type() == LocalFileLocation::Type::Partial);
2069     path = &node->local_.partial().path_;
2070     is_partial = true;
2071   }
2072 
2073   // TODO move file reading to another thread
2074   auto r_bytes = [&]() -> Result<string> {
2075     TRY_RESULT(fd, FileFd::open(*path, FileFd::Read));
2076     string data;
2077     data.resize(count);
2078     TRY_RESULT(read_bytes, fd.pread(data, offset));
2079     if (read_bytes != static_cast<size_t>(count)) {
2080       return Status::Error("Read less bytes than expected");
2081     }
2082     return std::move(data);
2083   }();
2084   if (r_bytes.is_error()) {
2085     LOG(INFO) << "Failed to read file bytes: " << r_bytes.error();
2086     if (--left_tries == 0 || !is_partial) {
2087       return promise.set_error(Status::Error(400, "Failed to read the file"));
2088     }
2089 
2090     // the temporary file could be moved from temp to persistent folder
2091     // we need to wait for the corresponding update and repeat the reading
2092     create_actor<SleepActor>("RepeatReadFilePartActor", 0.01,
2093                              PromiseCreator::lambda([actor_id = actor_id(this), file_id, offset, count, left_tries,
2094                                                      promise = std::move(promise)](Result<Unit> result) mutable {
2095                                send_closure(actor_id, &FileManager::read_file_part, file_id, offset, count, left_tries,
2096                                             std::move(promise));
2097                              }))
2098         .release();
2099     return;
2100   }
2101 
2102   auto result = td_api::make_object<td_api::filePart>();
2103   result->data_ = r_bytes.move_as_ok();
2104   promise.set_value(std::move(result));
2105 }
2106 
delete_file(FileId file_id,Promise<Unit> promise,const char * source)2107 void FileManager::delete_file(FileId file_id, Promise<Unit> promise, const char *source) {
2108   LOG(INFO) << "Trying to delete file " << file_id << " from " << source;
2109   auto node = get_sync_file_node(file_id);
2110   if (!node) {
2111     return promise.set_value(Unit());
2112   }
2113 
2114   auto file_view = FileView(node);
2115 
2116   // TODO review delete condition
2117   if (file_view.has_local_location()) {
2118     if (begins_with(file_view.local_location().path_, get_files_dir(file_view.get_type()))) {
2119       LOG(INFO) << "Unlink file " << file_id << " at " << file_view.local_location().path_;
2120       clear_from_pmc(node);
2121 
2122       context_->on_new_file(-file_view.size(), -file_view.get_allocated_local_size(), -1);
2123       unlink(file_view.local_location().path_).ignore();
2124       node->drop_local_location();
2125       try_flush_node(node, "delete_file 1");
2126     }
2127   } else {
2128     if (file_view.get_type() == FileType::Encrypted) {
2129       clear_from_pmc(node);
2130     }
2131     if (node->local_.type() == LocalFileLocation::Type::Partial) {
2132       LOG(INFO) << "Unlink partial file " << file_id << " at " << node->local_.partial().path_;
2133       unlink(node->local_.partial().path_).ignore();
2134       node->drop_local_location();
2135       try_flush_node(node, "delete_file 2");
2136     }
2137   }
2138 
2139   promise.set_value(Unit());
2140 }
2141 
download(FileId file_id,std::shared_ptr<DownloadCallback> callback,int32 new_priority,int64 offset,int64 limit)2142 void FileManager::download(FileId file_id, std::shared_ptr<DownloadCallback> callback, int32 new_priority, int64 offset,
2143                            int64 limit) {
2144   LOG(INFO) << "Download file " << file_id << " with priority " << new_priority;
2145   auto node = get_sync_file_node(file_id);
2146   if (!node) {
2147     LOG(INFO) << "File " << file_id << " not found";
2148     if (callback) {
2149       callback->on_download_error(file_id, Status::Error("File not found"));
2150     }
2151     return;
2152   }
2153 
2154   if (node->local_.type() == LocalFileLocation::Type::Full) {
2155     auto status = check_local_location(node);
2156     if (status.is_error()) {
2157       LOG(WARNING) << "Need to redownload file " << file_id << ": " << status.error();
2158     } else {
2159       LOG(INFO) << "File " << file_id << " is already downloaded";
2160       if (callback) {
2161         callback->on_download_ok(file_id);
2162       }
2163       return;
2164     }
2165   } else if (node->local_.type() == LocalFileLocation::Type::Partial) {
2166     auto status = check_local_location(node);
2167     if (status.is_error()) {
2168       LOG(WARNING) << "Need to download file " << file_id << " from beginning: " << status.error();
2169     }
2170   }
2171 
2172   FileView file_view(node);
2173   if (!file_view.can_download_from_server() && !file_view.can_generate()) {
2174     LOG(INFO) << "File " << file_id << " can't be downloaded";
2175     if (callback) {
2176       callback->on_download_error(file_id, Status::Error("Can't download or generate file"));
2177     }
2178     return;
2179   }
2180 
2181   if (new_priority == -1) {
2182     if (node->is_download_started_) {
2183       LOG(INFO) << "File " << file_id << " is being downloaded";
2184       return;
2185     }
2186     new_priority = 0;
2187   }
2188 
2189   LOG(INFO) << "Change download priority of file " << file_id << " to " << new_priority;
2190   node->set_download_offset(offset);
2191   node->set_download_limit(limit);
2192   auto *file_info = get_file_id_info(file_id);
2193   CHECK(new_priority == 0 || callback);
2194   if (file_info->download_callback_ != nullptr && file_info->download_callback_.get() != callback.get()) {
2195     // the callback will be destroyed soon and lost forever
2196     // this would be an error and should never happen, unless we cancel previous download query
2197     // in that case we send an error to the callback
2198     CHECK(new_priority == 0);
2199     file_info->download_callback_->on_download_error(file_id, Status::Error(200, "Canceled"));
2200   }
2201   file_info->download_priority_ = narrow_cast<int8>(new_priority);
2202   file_info->download_callback_ = std::move(callback);
2203   // TODO: send current progress?
2204 
2205   run_generate(node);
2206   run_download(node, true);
2207 
2208   try_flush_node(node, "download");
2209 }
2210 
run_download(FileNodePtr node,bool force_update_priority)2211 void FileManager::run_download(FileNodePtr node, bool force_update_priority) {
2212   int8 priority = 0;
2213   for (auto id : node->file_ids_) {
2214     auto *info = get_file_id_info(id);
2215     if (info->download_priority_ > priority) {
2216       priority = info->download_priority_;
2217     }
2218   }
2219 
2220   auto old_priority = node->download_priority_;
2221 
2222   if (priority == 0) {
2223     node->set_download_priority(priority);
2224     if (old_priority != 0) {
2225       LOG(INFO) << "Cancel downloading of file " << node->main_file_id_;
2226       do_cancel_download(node);
2227     }
2228     return;
2229   }
2230 
2231   if (node->need_load_from_pmc_) {
2232     LOG(INFO) << "Skip run_download, because file " << node->main_file_id_ << " needs to be loaded from PMC";
2233     return;
2234   }
2235   if (node->generate_id_) {
2236     LOG(INFO) << "Skip run_download, because file " << node->main_file_id_ << " is being generated";
2237     return;
2238   }
2239   auto file_view = FileView(node);
2240   if (!file_view.can_download_from_server()) {
2241     LOG(INFO) << "Skip run_download, because file " << node->main_file_id_ << " can't be downloaded from server";
2242     return;
2243   }
2244   node->set_download_priority(priority);
2245   bool need_update_offset = node->is_download_offset_dirty_;
2246   node->is_download_offset_dirty_ = false;
2247 
2248   bool need_update_limit = node->is_download_limit_dirty_;
2249   node->is_download_limit_dirty_ = false;
2250 
2251   if (old_priority != 0) {
2252     LOG(INFO) << "Update download offset and limits of file " << node->main_file_id_;
2253     CHECK(node->download_id_ != 0);
2254     if (force_update_priority || priority != old_priority) {
2255       send_closure(file_load_manager_, &FileLoadManager::update_priority, node->download_id_, priority);
2256     }
2257     if (need_update_limit || need_update_offset) {
2258       auto download_offset = node->download_offset_;
2259       auto download_limit = node->download_limit_;
2260       if (file_view.is_encrypted_any()) {
2261         CHECK(download_offset <= MAX_FILE_SIZE);
2262         CHECK(download_limit <= std::numeric_limits<int32>::max());
2263         download_limit += download_offset;
2264         download_offset = 0;
2265       }
2266       send_closure(file_load_manager_, &FileLoadManager::update_downloaded_part, node->download_id_, download_offset,
2267                    download_limit);
2268     }
2269     return;
2270   }
2271 
2272   CHECK(node->download_id_ == 0);
2273   CHECK(!node->file_ids_.empty());
2274   auto file_id = node->main_file_id_;
2275 
2276   if (node->need_reload_photo_ && file_view.may_reload_photo()) {
2277     LOG(INFO) << "Reload photo from file " << node->main_file_id_;
2278     QueryId id = queries_container_.create(Query{file_id, Query::Type::DownloadReloadDialog});
2279     node->download_id_ = id;
2280     context_->reload_photo(file_view.remote_location().get_source(),
2281                            PromiseCreator::lambda([id, actor_id = actor_id(this), file_id](Result<Unit> res) {
2282                              Status error;
2283                              if (res.is_ok()) {
2284                                error = Status::Error("FILE_DOWNLOAD_ID_INVALID");
2285                              } else {
2286                                error = res.move_as_error();
2287                              }
2288                              VLOG(file_references)
2289                                  << "Got result from reload photo for file " << file_id << ": " << error;
2290                              send_closure(actor_id, &FileManager::on_error, id, std::move(error));
2291                            }));
2292     node->need_reload_photo_ = false;
2293     return;
2294   }
2295 
2296   // If file reference is needed
2297   if (!file_view.has_active_download_remote_location()) {
2298     VLOG(file_references) << "Do not have valid file_reference for file " << file_id;
2299     QueryId id = queries_container_.create(Query{file_id, Query::Type::DownloadWaitFileReference});
2300     node->download_id_ = id;
2301     if (node->download_was_update_file_reference_) {
2302       on_error(id, Status::Error("Can't download file: have no valid file reference"));
2303       return;
2304     }
2305     node->download_was_update_file_reference_ = true;
2306 
2307     context_->repair_file_reference(
2308         file_id, PromiseCreator::lambda([id, actor_id = actor_id(this), file_id](Result<Unit> res) {
2309           Status error;
2310           if (res.is_ok()) {
2311             error = Status::Error("FILE_DOWNLOAD_RESTART_WITH_FILE_REFERENCE");
2312           } else {
2313             error = res.move_as_error();
2314           }
2315           VLOG(file_references) << "Got result from FileSourceManager for file " << file_id << ": " << error;
2316           send_closure(actor_id, &FileManager::on_error, id, std::move(error));
2317         }));
2318     return;
2319   }
2320 
2321   QueryId id = queries_container_.create(Query{file_id, Query::Type::Download});
2322   node->download_id_ = id;
2323   node->is_download_started_ = false;
2324   LOG(INFO) << "Run download of file " << file_id << " of size " << node->size_ << " from "
2325             << node->remote_.full.value() << " with suggested name " << node->suggested_path() << " and encyption key "
2326             << node->encryption_key_;
2327   auto download_offset = node->download_offset_;
2328   auto download_limit = node->download_limit_;
2329   if (file_view.is_encrypted_any()) {
2330     CHECK(download_offset <= MAX_FILE_SIZE);
2331     CHECK(download_limit <= std::numeric_limits<int32>::max());
2332     download_limit += download_offset;
2333     download_offset = 0;
2334   }
2335   send_closure(file_load_manager_, &FileLoadManager::download, id, node->remote_.full.value(), node->local_,
2336                node->size_, node->suggested_path(), node->encryption_key_, node->can_search_locally_, download_offset,
2337                download_limit, priority);
2338 }
2339 
2340 class FileManager::ForceUploadActor final : public Actor {
2341  public:
ForceUploadActor(FileManager * file_manager,FileId file_id,std::shared_ptr<FileManager::UploadCallback> callback,int32 new_priority,uint64 upload_order,bool prefer_small,ActorShared<> parent)2342   ForceUploadActor(FileManager *file_manager, FileId file_id, std::shared_ptr<FileManager::UploadCallback> callback,
2343                    int32 new_priority, uint64 upload_order, bool prefer_small, ActorShared<> parent)
2344       : file_manager_(file_manager)
2345       , file_id_(file_id)
2346       , callback_(std::move(callback))
2347       , new_priority_(new_priority)
2348       , upload_order_(upload_order)
2349       , prefer_small_(prefer_small)
2350       , parent_(std::move(parent)) {
2351   }
2352 
2353  private:
2354   FileManager *file_manager_;
2355   FileId file_id_;
2356   std::shared_ptr<FileManager::UploadCallback> callback_;
2357   int32 new_priority_;
2358   uint64 upload_order_;
2359   bool prefer_small_;
2360   ActorShared<> parent_;
2361   bool is_active_{false};
2362   int attempt_{0};
2363 
2364   class UploadCallback final : public FileManager::UploadCallback {
2365    public:
UploadCallback(ActorId<ForceUploadActor> callback)2366     explicit UploadCallback(ActorId<ForceUploadActor> callback) : callback_(std::move(callback)) {
2367     }
on_upload_ok(FileId file_id,tl_object_ptr<telegram_api::InputFile> input_file)2368     void on_upload_ok(FileId file_id, tl_object_ptr<telegram_api::InputFile> input_file) final {
2369       send_closure(std::move(callback_), &ForceUploadActor::on_upload_ok, std::move(input_file));
2370     }
2371 
on_upload_encrypted_ok(FileId file_id,tl_object_ptr<telegram_api::InputEncryptedFile> input_file)2372     void on_upload_encrypted_ok(FileId file_id, tl_object_ptr<telegram_api::InputEncryptedFile> input_file) final {
2373       send_closure(std::move(callback_), &ForceUploadActor::on_upload_encrypted_ok, std::move(input_file));
2374     }
2375 
on_upload_secure_ok(FileId file_id,tl_object_ptr<telegram_api::InputSecureFile> input_file)2376     void on_upload_secure_ok(FileId file_id, tl_object_ptr<telegram_api::InputSecureFile> input_file) final {
2377       send_closure(std::move(callback_), &ForceUploadActor::on_upload_secure_ok, std::move(input_file));
2378     }
2379 
on_upload_error(FileId file_id,Status error)2380     void on_upload_error(FileId file_id, Status error) final {
2381       send_closure(std::move(callback_), &ForceUploadActor::on_upload_error, std::move(error));
2382     }
~UploadCallback()2383     ~UploadCallback() final {
2384       if (callback_.empty()) {
2385         return;
2386       }
2387       send_closure(std::move(callback_), &ForceUploadActor::on_upload_error, Status::Error("Canceled"));
2388     }
2389 
2390    private:
2391     ActorId<ForceUploadActor> callback_;
2392   };
2393 
on_upload_ok(tl_object_ptr<telegram_api::InputFile> input_file)2394   void on_upload_ok(tl_object_ptr<telegram_api::InputFile> input_file) {
2395     is_active_ = false;
2396     if (input_file || is_ready()) {
2397       callback_->on_upload_ok(file_id_, std::move(input_file));
2398       on_ok();
2399     } else {
2400       loop();
2401     }
2402   }
2403 
on_upload_encrypted_ok(tl_object_ptr<telegram_api::InputEncryptedFile> input_file)2404   void on_upload_encrypted_ok(tl_object_ptr<telegram_api::InputEncryptedFile> input_file) {
2405     is_active_ = false;
2406     if (input_file || is_ready()) {
2407       callback_->on_upload_encrypted_ok(file_id_, std::move(input_file));
2408       on_ok();
2409     } else {
2410       loop();
2411     }
2412   }
2413 
on_upload_secure_ok(tl_object_ptr<telegram_api::InputSecureFile> input_file)2414   void on_upload_secure_ok(tl_object_ptr<telegram_api::InputSecureFile> input_file) {
2415     is_active_ = false;
2416     if (input_file || is_ready()) {
2417       callback_->on_upload_secure_ok(file_id_, std::move(input_file));
2418       on_ok();
2419     } else {
2420       loop();
2421     }
2422   }
2423 
is_ready() const2424   bool is_ready() const {
2425     return !G()->close_flag() && file_manager_->get_file_view(file_id_).has_active_upload_remote_location();
2426   }
2427 
on_ok()2428   void on_ok() {
2429     callback_.reset();
2430     send_closure(G()->file_manager(), &FileManager::on_force_reupload_success, file_id_);
2431     stop();
2432   }
2433 
on_upload_error(Status error)2434   void on_upload_error(Status error) {
2435     if (attempt_ == 2) {
2436       callback_->on_upload_error(file_id_, std::move(error));
2437       callback_.reset();
2438       stop();
2439     } else {
2440       is_active_ = false;
2441       loop();
2442     }
2443   }
2444 
create_callback()2445   auto create_callback() {
2446     return std::make_shared<UploadCallback>(actor_id(this));
2447   }
2448 
loop()2449   void loop() final {
2450     if (is_active_) {
2451       return;
2452     }
2453     if (G()->close_flag()) {
2454       return stop();
2455     }
2456 
2457     is_active_ = true;
2458     attempt_++;
2459     send_closure(G()->file_manager(), &FileManager::resume_upload, file_id_, std::vector<int>(), create_callback(),
2460                  new_priority_, upload_order_, attempt_ == 2, prefer_small_);
2461   }
2462 
tear_down()2463   void tear_down() final {
2464     if (callback_) {
2465       callback_->on_upload_error(file_id_, Status::Error("Canceled"));
2466     }
2467   }
2468 };
2469 
on_force_reupload_success(FileId file_id)2470 void FileManager::on_force_reupload_success(FileId file_id) {
2471   auto node = get_sync_file_node(file_id);
2472   CHECK(node);
2473   if (!node->remote_.is_full_alive) {  // do not update for multiple simultaneous uploads
2474     node->last_successful_force_reupload_time_ = Time::now();
2475   }
2476 }
2477 
resume_upload(FileId file_id,std::vector<int> bad_parts,std::shared_ptr<UploadCallback> callback,int32 new_priority,uint64 upload_order,bool force,bool prefer_small)2478 void FileManager::resume_upload(FileId file_id, std::vector<int> bad_parts, std::shared_ptr<UploadCallback> callback,
2479                                 int32 new_priority, uint64 upload_order, bool force, bool prefer_small) {
2480   auto node = get_sync_file_node(file_id);
2481   if (!node) {
2482     LOG(INFO) << "File " << file_id << " not found";
2483     if (callback) {
2484       callback->on_upload_error(file_id, Status::Error("File not found"));
2485     }
2486     return;
2487   }
2488 
2489   if (bad_parts.size() == 1 && bad_parts[0] == -1) {
2490     if (node->last_successful_force_reupload_time_ >= Time::now() - 60) {
2491       LOG(INFO) << "Recently reuploaded file " << file_id << ", do not try again";
2492       if (callback) {
2493         callback->on_upload_error(file_id, Status::Error("Failed to reupload file"));
2494       }
2495       return;
2496     }
2497 
2498     create_actor<ForceUploadActor>("ForceUploadActor", this, file_id, std::move(callback), new_priority, upload_order,
2499                                    prefer_small, context_->create_reference())
2500         .release();
2501     return;
2502   }
2503   LOG(INFO) << "Resume upload of file " << file_id << " with priority " << new_priority << " and force = " << force;
2504 
2505   if (force) {
2506     node->remote_.is_full_alive = false;
2507   }
2508   if (prefer_small) {
2509     node->upload_prefer_small_ = true;
2510   }
2511   if (node->upload_pause_ == file_id) {
2512     node->set_upload_pause(FileId());
2513   }
2514   FileView file_view(node);
2515   if (file_view.has_active_upload_remote_location() && file_view.get_type() != FileType::Thumbnail &&
2516       file_view.get_type() != FileType::EncryptedThumbnail && file_view.get_type() != FileType::Background) {
2517     LOG(INFO) << "File " << file_id << " is already uploaded";
2518     if (callback) {
2519       callback->on_upload_ok(file_id, nullptr);
2520     }
2521     return;
2522   }
2523 
2524   if (file_view.has_local_location()) {
2525     auto status = check_local_location(node);
2526     if (status.is_error()) {
2527       LOG(INFO) << "Full local location of file " << file_id << " for upload is invalid: " << status;
2528     }
2529   }
2530 
2531   if (!file_view.has_local_location() && !file_view.has_generate_location() && !file_view.has_alive_remote_location()) {
2532     LOG(INFO) << "File " << file_id << " can't be uploaded";
2533     if (callback) {
2534       callback->on_upload_error(file_id,
2535                                 Status::Error("Need full local (or generate, or inactive remote) location for upload"));
2536     }
2537     return;
2538   }
2539   if (file_view.get_type() == FileType::Thumbnail &&
2540       (!file_view.has_local_location() && file_view.can_download_from_server())) {
2541     // TODO
2542     if (callback) {
2543       callback->on_upload_error(file_id, Status::Error("Failed to upload thumbnail without local location"));
2544     }
2545     return;
2546   }
2547 
2548   LOG(INFO) << "Change upload priority of file " << file_id << " to " << new_priority;
2549   auto *file_info = get_file_id_info(file_id);
2550   CHECK(new_priority == 0 || callback);
2551   file_info->upload_order_ = upload_order;
2552   file_info->upload_priority_ = narrow_cast<int8>(new_priority);
2553   file_info->upload_callback_ = std::move(callback);
2554   // TODO: send current progress?
2555 
2556   run_generate(node);
2557   run_upload(node, std::move(bad_parts));
2558   try_flush_node(node, "resume_upload");
2559 }
2560 
delete_partial_remote_location(FileId file_id)2561 bool FileManager::delete_partial_remote_location(FileId file_id) {
2562   auto node = get_sync_file_node(file_id);
2563   if (!node) {
2564     LOG(INFO) << "Wrong file identifier " << file_id;
2565     return false;
2566   }
2567   if (node->upload_pause_ == file_id) {
2568     node->set_upload_pause(FileId());
2569   }
2570   if (node->remote_.is_full_alive) {
2571     LOG(INFO) << "File " << file_id << " is already uploaded";
2572     return true;
2573   }
2574 
2575   node->delete_partial_remote_location();
2576   auto *file_info = get_file_id_info(file_id);
2577   file_info->upload_priority_ = 0;
2578 
2579   if (node->local_.type() != LocalFileLocation::Type::Full) {
2580     LOG(INFO) << "Need full local location to upload file " << file_id;
2581     return false;
2582   }
2583 
2584   auto status = check_local_location(node);
2585   if (status.is_error()) {
2586     LOG(INFO) << "Need full local location to upload file " << file_id << ": " << status;
2587     return false;
2588   }
2589 
2590   run_upload(node, std::vector<int>());
2591   try_flush_node(node, "delete_partial_remote_location");
2592   return true;
2593 }
2594 
delete_file_reference(FileId file_id,Slice file_reference)2595 void FileManager::delete_file_reference(FileId file_id, Slice file_reference) {
2596   VLOG(file_references) << "Delete file reference of file " << file_id << " "
2597                         << tag("reference_base64", base64_encode(file_reference));
2598   auto node = get_sync_file_node(file_id);
2599   if (!node) {
2600     LOG(ERROR) << "Wrong file identifier " << file_id;
2601     return;
2602   }
2603   node->delete_file_reference(file_reference);
2604   auto remote = get_remote(file_id.get_remote());
2605   if (remote != nullptr) {
2606     VLOG(file_references) << "Do delete file reference of remote file " << file_id;
2607     if (remote->delete_file_reference(file_reference)) {
2608       VLOG(file_references) << "Successfully deleted file reference of remote file " << file_id;
2609       node->upload_was_update_file_reference_ = false;
2610       node->download_was_update_file_reference_ = false;
2611       node->on_pmc_changed();
2612     }
2613   }
2614   try_flush_node_pmc(node, "delete_file_reference");
2615 }
2616 
external_file_generate_write_part(int64 id,int32 offset,string data,Promise<> promise)2617 void FileManager::external_file_generate_write_part(int64 id, int32 offset, string data, Promise<> promise) {
2618   send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_write_part, id, offset,
2619                std::move(data), std::move(promise));
2620 }
2621 
external_file_generate_progress(int64 id,int32 expected_size,int32 local_prefix_size,Promise<> promise)2622 void FileManager::external_file_generate_progress(int64 id, int32 expected_size, int32 local_prefix_size,
2623                                                   Promise<> promise) {
2624   send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_progress, id, expected_size,
2625                local_prefix_size, std::move(promise));
2626 }
2627 
external_file_generate_finish(int64 id,Status status,Promise<> promise)2628 void FileManager::external_file_generate_finish(int64 id, Status status, Promise<> promise) {
2629   send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_finish, id, std::move(status),
2630                std::move(promise));
2631 }
2632 
run_generate(FileNodePtr node)2633 void FileManager::run_generate(FileNodePtr node) {
2634   if (node->need_load_from_pmc_) {
2635     LOG(INFO) << "Skip run_generate, because file " << node->main_file_id_ << " needs to be loaded from PMC";
2636     return;
2637   }
2638   FileView file_view(node);
2639   if (!file_view.can_generate()) {
2640     // LOG(INFO) << "Skip run_generate, because file " << node->main_file_id_ << " can't be generated";
2641     return;
2642   }
2643   if (file_view.has_local_location()) {
2644     LOG(INFO) << "Skip run_generate, because file " << node->main_file_id_ << " has local location";
2645     return;
2646   }
2647   if (file_view.can_download_from_server()) {
2648     LOG(INFO) << "Skip run_generate, because file " << node->main_file_id_ << " can be downloaded from server";
2649     return;
2650   }
2651 
2652   int8 download_priority = 0;
2653   int8 upload_priority = 0;
2654   FileId file_id = node->main_file_id_;
2655   for (auto id : node->file_ids_) {
2656     auto *info = get_file_id_info(id);
2657     if (info->download_priority_ > download_priority) {
2658       download_priority = info->download_priority_;
2659       if (download_priority > upload_priority) {
2660         file_id = id;
2661       }
2662     }
2663     if (info->upload_priority_ > upload_priority) {
2664       upload_priority = info->upload_priority_;
2665       if (upload_priority > download_priority) {
2666         file_id = id;
2667       }
2668     }
2669   }
2670 
2671   auto old_priority = node->generate_priority_;
2672   node->set_generate_priority(download_priority, upload_priority);
2673 
2674   if (node->generate_priority_ == 0) {
2675     if (old_priority != 0) {
2676       LOG(INFO) << "Cancel file " << file_id << " generation";
2677       do_cancel_generate(node);
2678     }
2679     return;
2680   }
2681 
2682   if (old_priority != 0) {
2683     LOG(INFO) << "TODO: change file " << file_id << " generation priority";
2684     return;
2685   }
2686 
2687   QueryId id = queries_container_.create(Query{file_id, Query::Type::Generate});
2688   node->generate_id_ = id;
2689   send_closure(file_generate_manager_, &FileGenerateManager::generate_file, id, *node->generate_, node->local_,
2690                node->suggested_path(), [file_manager = this, id] {
2691                  class Callback final : public FileGenerateCallback {
2692                    ActorId<FileManager> actor_;
2693                    uint64 query_id_;
2694 
2695                   public:
2696                    Callback(ActorId<FileManager> actor, QueryId id) : actor_(std::move(actor)), query_id_(id) {
2697                    }
2698                    void on_partial_generate(PartialLocalFileLocation partial_local, int32 expected_size) final {
2699                      send_closure(actor_, &FileManager::on_partial_generate, query_id_, std::move(partial_local),
2700                                   expected_size);
2701                    }
2702                    void on_ok(FullLocalFileLocation local) final {
2703                      send_closure(actor_, &FileManager::on_generate_ok, query_id_, std::move(local));
2704                    }
2705                    void on_error(Status error) final {
2706                      send_closure(actor_, &FileManager::on_error, query_id_, std::move(error));
2707                    }
2708                  };
2709                  return make_unique<Callback>(file_manager->actor_id(file_manager), id);
2710                }());
2711 
2712   LOG(INFO) << "File " << file_id << " generate request has sent to FileGenerateManager";
2713 }
2714 
run_upload(FileNodePtr node,std::vector<int> bad_parts)2715 void FileManager::run_upload(FileNodePtr node, std::vector<int> bad_parts) {
2716   int8 priority = 0;
2717   FileId file_id = node->main_file_id_;
2718   for (auto id : node->file_ids_) {
2719     auto *info = get_file_id_info(id);
2720     if (info->upload_priority_ > priority) {
2721       priority = info->upload_priority_;
2722       file_id = id;
2723     }
2724   }
2725 
2726   auto old_priority = node->upload_priority_;
2727 
2728   if (priority == 0) {
2729     node->set_upload_priority(priority);
2730     if (old_priority != 0) {
2731       LOG(INFO) << "Cancel file " << file_id << " uploading";
2732       do_cancel_upload(node);
2733     }
2734     return;
2735   }
2736 
2737   if (node->need_load_from_pmc_) {
2738     LOG(INFO) << "File " << node->main_file_id_ << " needs to be loaded from database before upload";
2739     return;
2740   }
2741   if (node->upload_pause_.is_valid()) {
2742     LOG(INFO) << "File " << node->main_file_id_ << " upload is paused: " << node->upload_pause_;
2743     return;
2744   }
2745 
2746   FileView file_view(node);
2747   if (!file_view.has_local_location() && !file_view.has_remote_location()) {
2748     if (node->get_by_hash_ || node->generate_id_ == 0 || !node->generate_was_update_) {
2749       LOG(INFO) << "Have no local location for file: get_by_hash = " << node->get_by_hash_
2750                 << ", generate_id = " << node->generate_id_ << ", generate_was_update = " << node->generate_was_update_;
2751       return;
2752     }
2753     if (file_view.has_generate_location() && file_view.generate_location().file_type_ == FileType::Secure) {
2754       // Can't upload secure file before its size is known
2755       LOG(INFO) << "Can't upload secure file " << node->main_file_id_ << " before it's size is known";
2756       return;
2757     }
2758   }
2759 
2760   node->set_upload_priority(priority);
2761 
2762   // create encryption key if necessary
2763   if (((file_view.has_generate_location() && file_view.generate_location().file_type_ == FileType::Encrypted) ||
2764        (file_view.has_local_location() && file_view.local_location().file_type_ == FileType::Encrypted)) &&
2765       file_view.encryption_key().empty()) {
2766     CHECK(!node->file_ids_.empty());
2767     bool success = set_encryption_key(node->file_ids_[0], FileEncryptionKey::create());
2768     LOG_IF(FATAL, !success) << "Failed to set encryption key for file " << file_id;
2769   }
2770 
2771   // create encryption key if necessary
2772   if (file_view.has_local_location() && file_view.local_location().file_type_ == FileType::Secure &&
2773       file_view.encryption_key().empty()) {
2774     CHECK(!node->file_ids_.empty());
2775     bool success = set_encryption_key(node->file_ids_[0], FileEncryptionKey::create_secure_key());
2776     LOG_IF(FATAL, !success) << "Failed to set encryption key for file " << file_id;
2777   }
2778 
2779   if (old_priority != 0) {
2780     LOG(INFO) << "File " << file_id << " is already uploading";
2781     CHECK(node->upload_id_ != 0);
2782     send_closure(file_load_manager_, &FileLoadManager::update_priority, node->upload_id_, narrow_cast<int8>(-priority));
2783     return;
2784   }
2785 
2786   CHECK(node->upload_id_ == 0);
2787   if (file_view.has_alive_remote_location() && !file_view.has_active_upload_remote_location() &&
2788       file_view.get_type() != FileType::Thumbnail && file_view.get_type() != FileType::EncryptedThumbnail &&
2789       file_view.get_type() != FileType::Background) {
2790     QueryId id = queries_container_.create(Query{file_id, Query::Type::UploadWaitFileReference});
2791     node->upload_id_ = id;
2792     if (node->upload_was_update_file_reference_) {
2793       on_error(id, Status::Error("Can't upload file: have no valid file reference"));
2794       return;
2795     }
2796     node->upload_was_update_file_reference_ = true;
2797 
2798     context_->repair_file_reference(
2799         node->main_file_id_, PromiseCreator::lambda([id, actor_id = actor_id(this)](Result<Unit> res) {
2800           send_closure(actor_id, &FileManager::on_error, id, Status::Error("FILE_UPLOAD_RESTART_WITH_FILE_REFERENCE"));
2801         }));
2802     return;
2803   }
2804 
2805   if (!node->remote_.partial && node->get_by_hash_) {
2806     LOG(INFO) << "Get file " << node->main_file_id_ << " by hash";
2807     QueryId id = queries_container_.create(Query{file_id, Query::Type::UploadByHash});
2808     node->upload_id_ = id;
2809 
2810     send_closure(file_load_manager_, &FileLoadManager::upload_by_hash, id, node->local_.full(), node->size_,
2811                  narrow_cast<int8>(-priority));
2812     return;
2813   }
2814 
2815   auto new_priority = narrow_cast<int8>(bad_parts.empty() ? -priority : priority);
2816   td::remove_if(bad_parts, [](auto part_id) { return part_id < 0; });
2817 
2818   auto expected_size = file_view.expected_size(true);
2819   if (node->upload_prefer_small_ && (10 << 20) < expected_size && expected_size < (30 << 20)) {
2820     expected_size = 10 << 20;
2821   }
2822 
2823   QueryId id = queries_container_.create(Query{file_id, Query::Type::Upload});
2824   node->upload_id_ = id;
2825   send_closure(file_load_manager_, &FileLoadManager::upload, id, node->local_, node->remote_.partial_or_empty(),
2826                expected_size, node->encryption_key_, new_priority, std::move(bad_parts));
2827 
2828   LOG(INFO) << "File " << file_id << " upload request has sent to FileLoadManager";
2829 }
2830 
upload(FileId file_id,std::shared_ptr<UploadCallback> callback,int32 new_priority,uint64 upload_order)2831 void FileManager::upload(FileId file_id, std::shared_ptr<UploadCallback> callback, int32 new_priority,
2832                          uint64 upload_order) {
2833   return resume_upload(file_id, std::vector<int>(), std::move(callback), new_priority, upload_order);
2834 }
2835 
cancel_upload(FileId file_id)2836 void FileManager::cancel_upload(FileId file_id) {
2837   return resume_upload(file_id, std::vector<int>(), nullptr, 0, 0);
2838 }
2839 
is_document_type(FileType type)2840 static bool is_document_type(FileType type) {
2841   return type == FileType::Document || type == FileType::Sticker || type == FileType::Audio ||
2842          type == FileType::Animation || type == FileType::Background || type == FileType::DocumentAsFile;
2843 }
2844 
is_background_type(FileType type)2845 static bool is_background_type(FileType type) {
2846   return type == FileType::Wallpaper || type == FileType::Background;
2847 }
2848 
from_persistent_id(CSlice persistent_id,FileType file_type)2849 Result<FileId> FileManager::from_persistent_id(CSlice persistent_id, FileType file_type) {
2850   if (persistent_id.find('.') != string::npos) {
2851     auto r_http_url = parse_url(persistent_id);
2852     if (r_http_url.is_error()) {
2853       return Status::Error(400, PSLICE() << "Invalid file HTTP URL specified: " << r_http_url.error().message());
2854     }
2855     auto url = r_http_url.ok().get_url();
2856     if (!clean_input_string(url)) {
2857       return Status::Error(400, "URL must be in UTF-8");
2858     }
2859     return register_url(std::move(url), file_type, FileLocationSource::FromUser, DialogId());
2860   }
2861 
2862   auto r_binary = base64url_decode(persistent_id);
2863   if (r_binary.is_error()) {
2864     return Status::Error(400, PSLICE() << "Wrong remote file identifier specified: " << r_binary.error().message());
2865   }
2866   auto binary = r_binary.move_as_ok();
2867   if (binary.empty()) {
2868     return Status::Error(400, "Remote file identifier can't be empty");
2869   }
2870   if (binary.back() == FileNode::PERSISTENT_ID_VERSION_OLD) {
2871     return from_persistent_id_v2(binary, file_type);
2872   }
2873   if (binary.back() == FileNode::PERSISTENT_ID_VERSION) {
2874     return from_persistent_id_v3(binary, file_type);
2875   }
2876   if (binary.back() == FileNode::PERSISTENT_ID_VERSION_MAP) {
2877     return from_persistent_id_map(binary, file_type);
2878   }
2879   return Status::Error(400, "Wrong remote file identifier specified: can't unserialize it. Wrong last symbol");
2880 }
2881 
from_persistent_id_map(Slice binary,FileType file_type)2882 Result<FileId> FileManager::from_persistent_id_map(Slice binary, FileType file_type) {
2883   binary.remove_suffix(1);
2884   auto decoded_binary = zero_decode(binary);
2885   FullGenerateFileLocation generate_location;
2886   auto status = unserialize(generate_location, decoded_binary);
2887   if (status.is_error()) {
2888     return Status::Error(400, "Wrong remote file identifier specified: can't unserialize it");
2889   }
2890   auto real_file_type = generate_location.file_type_;
2891   if ((real_file_type != file_type && file_type != FileType::Temp) ||
2892       (real_file_type != FileType::Thumbnail && real_file_type != FileType::EncryptedThumbnail)) {
2893     return Status::Error(400, "Type of file mismatch");
2894   }
2895   if (!begins_with(generate_location.conversion_, "#map#")) {
2896     return Status::Error(400, "Unexpected conversion type");
2897   }
2898   FileData data;
2899   data.generate_ = make_unique<FullGenerateFileLocation>(std::move(generate_location));
2900   return register_file(std::move(data), FileLocationSource::FromUser, "from_persistent_id_map", false).move_as_ok();
2901 }
2902 
from_persistent_id_v23(Slice binary,FileType file_type,int32 version)2903 Result<FileId> FileManager::from_persistent_id_v23(Slice binary, FileType file_type, int32 version) {
2904   if (version < 0 || version >= static_cast<int32>(Version::Next)) {
2905     return Status::Error(400, "Invalid remote file identifier");
2906   }
2907   auto decoded_binary = zero_decode(binary);
2908   FullRemoteFileLocation remote_location;
2909   log_event::WithVersion<TlParser> parser(decoded_binary);
2910   parser.set_version(version);
2911   parse(remote_location, parser);
2912   parser.fetch_end();
2913   auto status = parser.get_status();
2914   if (status.is_error()) {
2915     return Status::Error(400, "Wrong remote file identifier specified: can't unserialize it");
2916   }
2917   auto &real_file_type = remote_location.file_type_;
2918   if (is_document_type(real_file_type) && is_document_type(file_type)) {
2919     real_file_type = file_type;
2920   } else if (is_background_type(real_file_type) && is_background_type(file_type)) {
2921     // type of file matches, but real type is in the stored remote location
2922   } else if (real_file_type != file_type && file_type != FileType::Temp) {
2923     return Status::Error(400, "Type of file mismatch");
2924   }
2925   FileData data;
2926   data.remote_ = RemoteFileLocation(std::move(remote_location));
2927   auto file_id =
2928       register_file(std::move(data), FileLocationSource::FromUser, "from_persistent_id_v23", false).move_as_ok();
2929   return file_id;
2930 }
2931 
from_persistent_id_v2(Slice binary,FileType file_type)2932 Result<FileId> FileManager::from_persistent_id_v2(Slice binary, FileType file_type) {
2933   binary.remove_suffix(1);
2934   return from_persistent_id_v23(binary, file_type, 0);
2935 }
2936 
from_persistent_id_v3(Slice binary,FileType file_type)2937 Result<FileId> FileManager::from_persistent_id_v3(Slice binary, FileType file_type) {
2938   binary.remove_suffix(1);
2939   if (binary.empty()) {
2940     return Status::Error(400, "Invalid remote file identifier");
2941   }
2942   int32 version = static_cast<uint8>(binary.back());
2943   binary.remove_suffix(1);
2944   return from_persistent_id_v23(binary, file_type, version);
2945 }
2946 
get_file_view(FileId file_id) const2947 FileView FileManager::get_file_view(FileId file_id) const {
2948   auto file_node = get_file_node(file_id);
2949   if (!file_node) {
2950     return FileView();
2951   }
2952   return FileView(file_node);
2953 }
2954 
get_sync_file_view(FileId file_id)2955 FileView FileManager::get_sync_file_view(FileId file_id) {
2956   auto file_node = get_sync_file_node(file_id);
2957   if (!file_node) {
2958     return FileView();
2959   }
2960   return FileView(file_node);
2961 }
2962 
get_file_object(FileId file_id,bool with_main_file_id)2963 td_api::object_ptr<td_api::file> FileManager::get_file_object(FileId file_id, bool with_main_file_id) {
2964   auto file_view = get_sync_file_view(file_id);
2965 
2966   if (file_view.empty()) {
2967     return td_api::make_object<td_api::file>(0, 0, 0, td_api::make_object<td_api::localFile>(),
2968                                              td_api::make_object<td_api::remoteFile>());
2969   }
2970 
2971   string persistent_file_id = file_view.get_persistent_file_id();
2972   string unique_file_id = file_view.get_unique_file_id();
2973   bool is_uploading_completed = !persistent_file_id.empty();
2974   auto size = narrow_cast<int32>(file_view.size());
2975   auto expected_size = narrow_cast<int32>(file_view.expected_size());
2976   auto download_offset = narrow_cast<int32>(file_view.download_offset());
2977   auto local_prefix_size = narrow_cast<int32>(file_view.local_prefix_size());
2978   auto local_total_size = narrow_cast<int32>(file_view.local_total_size());
2979   auto remote_size = narrow_cast<int32>(file_view.remote_size());
2980   string path = file_view.path();
2981   bool can_be_downloaded = file_view.can_download_from_server() || file_view.can_generate();
2982   bool can_be_deleted = file_view.can_delete();
2983 
2984   auto result_file_id = file_id;
2985   auto *file_info = get_file_id_info(result_file_id);
2986   if (with_main_file_id) {
2987     if (!file_info->send_updates_flag_) {
2988       result_file_id = file_view.file_id();
2989     }
2990     file_info = get_file_id_info(file_view.file_id());
2991   }
2992   file_info->send_updates_flag_ = true;
2993   VLOG(update_file) << "Send file " << file_id << " as " << result_file_id << " and update send_updates_flag_ for file "
2994                     << (with_main_file_id ? file_view.file_id() : result_file_id);
2995 
2996   return td_api::make_object<td_api::file>(
2997       result_file_id.get(), size, expected_size,
2998       td_api::make_object<td_api::localFile>(std::move(path), can_be_downloaded, can_be_deleted,
2999                                              file_view.is_downloading(), file_view.has_local_location(),
3000                                              download_offset, local_prefix_size, local_total_size),
3001       td_api::make_object<td_api::remoteFile>(std::move(persistent_file_id), std::move(unique_file_id),
3002                                               file_view.is_uploading(), is_uploading_completed, remote_size));
3003 }
3004 
get_file_ids_object(const vector<FileId> & file_ids,bool with_main_file_id)3005 vector<int32> FileManager::get_file_ids_object(const vector<FileId> &file_ids, bool with_main_file_id) {
3006   return transform(file_ids, [this, with_main_file_id](FileId file_id) {
3007     auto file_view = get_sync_file_view(file_id);
3008     auto result_file_id = file_id;
3009     auto *file_info = get_file_id_info(result_file_id);
3010     if (with_main_file_id) {
3011       if (!file_info->sent_file_id_flag_ && !file_info->send_updates_flag_) {
3012         result_file_id = file_view.file_id();
3013       }
3014       file_info = get_file_id_info(file_view.file_id());
3015     }
3016     file_info->sent_file_id_flag_ = true;
3017 
3018     return result_file_id.get();
3019   });
3020 }
3021 
check_input_file_id(FileType type,Result<FileId> result,bool is_encrypted,bool allow_zero,bool is_secure)3022 Result<FileId> FileManager::check_input_file_id(FileType type, Result<FileId> result, bool is_encrypted,
3023                                                 bool allow_zero, bool is_secure) {
3024   TRY_RESULT(file_id, std::move(result));
3025   if (allow_zero && !file_id.is_valid()) {
3026     return FileId();
3027   }
3028 
3029   auto file_node = get_sync_file_node(file_id);  // we need full data about sent files
3030   if (!file_node) {
3031     return Status::Error(400, "File not found");
3032   }
3033   auto file_view = FileView(file_node);
3034   FileType real_type = file_view.get_type();
3035   LOG(INFO) << "Checking file " << file_id << " of type " << type << "/" << real_type;
3036   if (!is_encrypted && !is_secure) {
3037     if (real_type != type && !(real_type == FileType::Temp && file_view.has_url()) &&
3038         !(is_document_type(real_type) && is_document_type(type)) &&
3039         !(is_background_type(real_type) && is_background_type(type))) {
3040       // TODO: send encrypted file to unencrypted chat
3041       return Status::Error(400, "Type of file mismatch");
3042     }
3043   }
3044 
3045   if (!file_view.has_remote_location()) {
3046     // TODO why not return file_id here? We will dup it anyway
3047     // But it will not be duped if has_input_media(), so for now we can't return main_file_id
3048 
3049     if (file_view.has_url() && !is_encrypted) {
3050       // URLs in non-secret chats never needs to be reuploaded, so they don't need to be duped
3051       return file_node->main_file_id_;
3052     }
3053     return dup_file_id(file_id);
3054   }
3055 
3056   int32 remote_id = file_id.get_remote();
3057   if (remote_id == 0) {
3058     RemoteInfo info{file_view.remote_location(), FileLocationSource::FromUser, file_id};
3059     remote_id = remote_location_info_.add(info);
3060     if (remote_location_info_.get(remote_id).file_id_ == file_id) {
3061       get_file_id_info(file_id)->pin_flag_ = true;
3062     }
3063   }
3064   return FileId(file_node->main_file_id_.get(), remote_id);
3065 }
3066 
get_input_thumbnail_file_id(const tl_object_ptr<td_api::InputFile> & thumbnail_input_file,DialogId owner_dialog_id,bool is_encrypted)3067 Result<FileId> FileManager::get_input_thumbnail_file_id(const tl_object_ptr<td_api::InputFile> &thumbnail_input_file,
3068                                                         DialogId owner_dialog_id, bool is_encrypted) {
3069   if (thumbnail_input_file == nullptr) {
3070     return Status::Error(400, "inputThumbnail not specified");
3071   }
3072 
3073   switch (thumbnail_input_file->get_id()) {
3074     case td_api::inputFileLocal::ID: {
3075       const string &path = static_cast<const td_api::inputFileLocal *>(thumbnail_input_file.get())->path_;
3076       return register_local(
3077           FullLocalFileLocation(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail, path, 0),
3078           owner_dialog_id, 0, false);
3079     }
3080     case td_api::inputFileId::ID:
3081       return Status::Error(400, "InputFileId is not supported for thumbnails");
3082     case td_api::inputFileRemote::ID:
3083       return Status::Error(400, "InputFileRemote is not supported for thumbnails");
3084     case td_api::inputFileGenerated::ID: {
3085       auto *generated_thumbnail = static_cast<const td_api::inputFileGenerated *>(thumbnail_input_file.get());
3086       return register_generate(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail,
3087                                FileLocationSource::FromUser, generated_thumbnail->original_path_,
3088                                generated_thumbnail->conversion_, owner_dialog_id, generated_thumbnail->expected_size_);
3089     }
3090     default:
3091       UNREACHABLE();
3092       return Status::Error(500, "Unreachable");
3093   }
3094 }
3095 
get_input_file_id(FileType type,const tl_object_ptr<td_api::InputFile> & file,DialogId owner_dialog_id,bool allow_zero,bool is_encrypted,bool get_by_hash,bool is_secure)3096 Result<FileId> FileManager::get_input_file_id(FileType type, const tl_object_ptr<td_api::InputFile> &file,
3097                                               DialogId owner_dialog_id, bool allow_zero, bool is_encrypted,
3098                                               bool get_by_hash, bool is_secure) {
3099   if (file == nullptr) {
3100     if (allow_zero) {
3101       return FileId();
3102     }
3103     return Status::Error(400, "InputFile is not specified");
3104   }
3105 
3106   if (is_encrypted || is_secure) {
3107     get_by_hash = false;
3108   }
3109 
3110   auto new_type = is_encrypted ? FileType::Encrypted : (is_secure ? FileType::Secure : type);
3111 
3112   auto r_file_id = [&]() -> Result<FileId> {
3113     switch (file->get_id()) {
3114       case td_api::inputFileLocal::ID: {
3115         const string &path = static_cast<const td_api::inputFileLocal *>(file.get())->path_;
3116         if (allow_zero && path.empty()) {
3117           return FileId();
3118         }
3119         string hash;
3120         if (G()->shared_config().get_option_boolean("reuse_uploaded_photos_by_hash") && new_type == FileType::Photo) {
3121           auto r_stat = stat(path);
3122           if (r_stat.is_ok() && r_stat.ok().size_ > 0 && r_stat.ok().size_ < 11000000) {
3123             auto r_file_content = read_file_str(path, r_stat.ok().size_);
3124             if (r_file_content.is_ok()) {
3125               hash = sha256(r_file_content.ok());
3126               auto it = file_hash_to_file_id_.find(hash);
3127               if (it != file_hash_to_file_id_.end()) {
3128                 auto file_view = get_file_view(it->second);
3129                 if (!file_view.empty() && file_view.has_remote_location() && !file_view.remote_location().is_web()) {
3130                   return it->second;
3131                 }
3132               }
3133             }
3134           }
3135         }
3136         TRY_RESULT(file_id, register_local(FullLocalFileLocation(new_type, path, 0), owner_dialog_id, 0, get_by_hash));
3137         if (!hash.empty()) {
3138           file_hash_to_file_id_[hash] = file_id;
3139         }
3140         return file_id;
3141       }
3142       case td_api::inputFileId::ID: {
3143         FileId file_id(static_cast<const td_api::inputFileId *>(file.get())->id_, 0);
3144         if (!file_id.is_valid()) {
3145           return FileId();
3146         }
3147         return file_id;
3148       }
3149       case td_api::inputFileRemote::ID: {
3150         const string &file_persistent_id = static_cast<const td_api::inputFileRemote *>(file.get())->id_;
3151         if (allow_zero && file_persistent_id.empty()) {
3152           return FileId();
3153         }
3154         return from_persistent_id(file_persistent_id, type);
3155       }
3156       case td_api::inputFileGenerated::ID: {
3157         auto *generated_file = static_cast<const td_api::inputFileGenerated *>(file.get());
3158         return register_generate(new_type, FileLocationSource::FromUser, generated_file->original_path_,
3159                                  generated_file->conversion_, owner_dialog_id, generated_file->expected_size_);
3160       }
3161       default:
3162         UNREACHABLE();
3163         return Status::Error(500, "Unreachable");
3164     }
3165   }();
3166 
3167   return check_input_file_id(type, std::move(r_file_id), is_encrypted, allow_zero, is_secure);
3168 }
3169 
get_map_thumbnail_file_id(Location location,int32 zoom,int32 width,int32 height,int32 scale,DialogId owner_dialog_id)3170 Result<FileId> FileManager::get_map_thumbnail_file_id(Location location, int32 zoom, int32 width, int32 height,
3171                                                       int32 scale, DialogId owner_dialog_id) {
3172   if (!location.is_valid_map_point()) {
3173     return Status::Error(400, "Invalid location specified");
3174   }
3175   if (zoom < 13 || zoom > 20) {
3176     return Status::Error(400, "Wrong zoom");
3177   }
3178   if (width < 16 || width > 1024) {
3179     return Status::Error(400, "Wrong width");
3180   }
3181   if (height < 16 || height > 1024) {
3182     return Status::Error(400, "Wrong height");
3183   }
3184   if (scale < 1 || scale > 3) {
3185     return Status::Error(400, "Wrong scale");
3186   }
3187 
3188   const double PI = 3.14159265358979323846;
3189   double sin_latitude = std::sin(location.get_latitude() * PI / 180);
3190   int32 size = 256 * (1 << zoom);
3191   auto x = static_cast<int32>((location.get_longitude() + 180) / 360 * size);
3192   auto y = static_cast<int32>((0.5 - std::log((1 + sin_latitude) / (1 - sin_latitude)) / (4 * PI)) * size);
3193   x = clamp(x, 0, size - 1);  // just in case
3194   y = clamp(y, 0, size - 1);  // just in case
3195 
3196   string conversion = PSTRING() << "#map#" << zoom << '#' << x << '#' << y << '#' << width << '#' << height << '#'
3197                                 << scale << '#';
3198   return register_generate(
3199       owner_dialog_id.get_type() == DialogType::SecretChat ? FileType::EncryptedThumbnail : FileType::Thumbnail,
3200       FileLocationSource::FromUser, string(), std::move(conversion), owner_dialog_id, 0);
3201 }
3202 
guess_file_type(const tl_object_ptr<td_api::InputFile> & file)3203 FileType FileManager::guess_file_type(const tl_object_ptr<td_api::InputFile> &file) {
3204   if (file == nullptr) {
3205     return FileType::Temp;
3206   }
3207 
3208   auto guess_file_type_by_path = [](const string &file_path) {
3209     PathView path_view(file_path);
3210     auto file_name = path_view.file_name();
3211     auto extension = path_view.extension();
3212     if (extension == "jpg" || extension == "jpeg") {
3213       return FileType::Photo;
3214     }
3215     if (extension == "ogg" || extension == "oga" || extension == "opus") {
3216       return FileType::VoiceNote;
3217     }
3218     if (extension == "3gp" || extension == "mov") {
3219       return FileType::Video;
3220     }
3221     if (extension == "mp3" || extension == "mpeg3" || extension == "m4a") {
3222       return FileType::Audio;
3223     }
3224     if (extension == "webp" || extension == "tgs") {
3225       return FileType::Sticker;
3226     }
3227     if (extension == "gif") {
3228       return FileType::Animation;
3229     }
3230     if (extension == "mp4" || extension == "mpeg4") {
3231       return to_lower(file_name).find("-gif-") != string::npos ? FileType::Animation : FileType::Video;
3232     }
3233     return FileType::Document;
3234   };
3235 
3236   switch (file->get_id()) {
3237     case td_api::inputFileLocal::ID:
3238       return guess_file_type_by_path(static_cast<const td_api::inputFileLocal *>(file.get())->path_);
3239     case td_api::inputFileId::ID: {
3240       FileId file_id(static_cast<const td_api::inputFileId *>(file.get())->id_, 0);
3241       auto file_view = get_file_view(file_id);
3242       if (file_view.empty()) {
3243         return FileType::Temp;
3244       }
3245       return file_view.get_type();
3246     }
3247     case td_api::inputFileRemote::ID: {
3248       const string &file_persistent_id = static_cast<const td_api::inputFileRemote *>(file.get())->id_;
3249       Result<FileId> r_file_id = from_persistent_id(file_persistent_id, FileType::Temp);
3250       if (r_file_id.is_error()) {
3251         return FileType::Temp;
3252       }
3253       auto file_view = get_file_view(r_file_id.ok());
3254       if (file_view.empty()) {
3255         return FileType::Temp;
3256       }
3257       return file_view.get_type();
3258     }
3259     case td_api::inputFileGenerated::ID:
3260       return guess_file_type_by_path(static_cast<const td_api::inputFileGenerated *>(file.get())->original_path_);
3261     default:
3262       UNREACHABLE();
3263       return FileType::Temp;
3264   }
3265 }
3266 
get_input_documents(const vector<FileId> & file_ids)3267 vector<tl_object_ptr<telegram_api::InputDocument>> FileManager::get_input_documents(const vector<FileId> &file_ids) {
3268   vector<tl_object_ptr<telegram_api::InputDocument>> result;
3269   result.reserve(file_ids.size());
3270   for (auto file_id : file_ids) {
3271     auto file_view = get_file_view(file_id);
3272     CHECK(!file_view.empty());
3273     CHECK(file_view.has_remote_location());
3274     CHECK(!file_view.remote_location().is_web());
3275     result.push_back(file_view.remote_location().as_input_document());
3276   }
3277   return result;
3278 }
3279 
extract_was_uploaded(const tl_object_ptr<telegram_api::InputMedia> & input_media)3280 bool FileManager::extract_was_uploaded(const tl_object_ptr<telegram_api::InputMedia> &input_media) {
3281   if (input_media == nullptr) {
3282     return false;
3283   }
3284 
3285   auto input_media_id = input_media->get_id();
3286   return input_media_id == telegram_api::inputMediaUploadedPhoto::ID ||
3287          input_media_id == telegram_api::inputMediaUploadedDocument::ID;
3288 }
3289 
extract_was_thumbnail_uploaded(const tl_object_ptr<telegram_api::InputMedia> & input_media)3290 bool FileManager::extract_was_thumbnail_uploaded(const tl_object_ptr<telegram_api::InputMedia> &input_media) {
3291   if (input_media == nullptr || input_media->get_id() != telegram_api::inputMediaUploadedDocument::ID) {
3292     return false;
3293   }
3294 
3295   return static_cast<const telegram_api::inputMediaUploadedDocument *>(input_media.get())->thumb_ != nullptr;
3296 }
3297 
extract_file_reference(const tl_object_ptr<telegram_api::InputMedia> & input_media)3298 string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputMedia> &input_media) {
3299   if (input_media == nullptr) {
3300     return string();
3301   }
3302 
3303   switch (input_media->get_id()) {
3304     case telegram_api::inputMediaDocument::ID:
3305       return extract_file_reference(static_cast<const telegram_api::inputMediaDocument *>(input_media.get())->id_);
3306     case telegram_api::inputMediaPhoto::ID:
3307       return extract_file_reference(static_cast<const telegram_api::inputMediaPhoto *>(input_media.get())->id_);
3308     default:
3309       return string();
3310   }
3311 }
3312 
extract_file_reference(const tl_object_ptr<telegram_api::InputDocument> & input_document)3313 string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputDocument> &input_document) {
3314   if (input_document == nullptr || input_document->get_id() != telegram_api::inputDocument::ID) {
3315     return string();
3316   }
3317 
3318   return static_cast<const telegram_api::inputDocument *>(input_document.get())->file_reference_.as_slice().str();
3319 }
3320 
extract_file_reference(const tl_object_ptr<telegram_api::InputPhoto> & input_photo)3321 string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputPhoto> &input_photo) {
3322   if (input_photo == nullptr || input_photo->get_id() != telegram_api::inputPhoto::ID) {
3323     return string();
3324   }
3325 
3326   return static_cast<const telegram_api::inputPhoto *>(input_photo.get())->file_reference_.as_slice().str();
3327 }
3328 
extract_was_uploaded(const tl_object_ptr<telegram_api::InputChatPhoto> & input_chat_photo)3329 bool FileManager::extract_was_uploaded(const tl_object_ptr<telegram_api::InputChatPhoto> &input_chat_photo) {
3330   return input_chat_photo != nullptr && input_chat_photo->get_id() == telegram_api::inputChatUploadedPhoto::ID;
3331 }
3332 
extract_file_reference(const tl_object_ptr<telegram_api::InputChatPhoto> & input_chat_photo)3333 string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputChatPhoto> &input_chat_photo) {
3334   if (input_chat_photo == nullptr || input_chat_photo->get_id() != telegram_api::inputChatPhoto::ID) {
3335     return string();
3336   }
3337 
3338   return extract_file_reference(static_cast<const telegram_api::inputChatPhoto *>(input_chat_photo.get())->id_);
3339 }
3340 
next_file_id()3341 FileId FileManager::next_file_id() {
3342   if (!empty_file_ids_.empty()) {
3343     auto res = empty_file_ids_.back();
3344     empty_file_ids_.pop_back();
3345     return FileId{res, 0};
3346   }
3347   FileId res(static_cast<int32>(file_id_info_.size()), 0);
3348   // LOG(ERROR) << "NEXT file_id " << res;
3349   file_id_info_.push_back({});
3350   return res;
3351 }
3352 
next_file_node_id()3353 FileManager::FileNodeId FileManager::next_file_node_id() {
3354   auto res = static_cast<FileNodeId>(file_nodes_.size());
3355   file_nodes_.emplace_back(nullptr);
3356   return res;
3357 }
3358 
on_start_download(QueryId query_id)3359 void FileManager::on_start_download(QueryId query_id) {
3360   if (is_closed_) {
3361     return;
3362   }
3363 
3364   auto query = queries_container_.get(query_id);
3365   CHECK(query != nullptr);
3366 
3367   auto file_id = query->file_id_;
3368   auto file_node = get_file_node(file_id);
3369   LOG(DEBUG) << "Receive on_start_download for file " << file_id;
3370   if (!file_node) {
3371     return;
3372   }
3373   if (file_node->download_id_ != query_id) {
3374     return;
3375   }
3376 
3377   LOG(DEBUG) << "Start to download part of file " << file_id;
3378   file_node->is_download_started_ = true;
3379 }
3380 
on_partial_download(QueryId query_id,PartialLocalFileLocation partial_local,int64 ready_size,int64 size)3381 void FileManager::on_partial_download(QueryId query_id, PartialLocalFileLocation partial_local, int64 ready_size,
3382                                       int64 size) {
3383   if (is_closed_) {
3384     return;
3385   }
3386 
3387   auto query = queries_container_.get(query_id);
3388   CHECK(query != nullptr);
3389 
3390   auto file_id = query->file_id_;
3391   auto file_node = get_file_node(file_id);
3392   LOG(DEBUG) << "Receive on_partial_download for file " << file_id << " with " << partial_local
3393              << ", ready_size = " << ready_size << " and size = " << size;
3394   if (!file_node) {
3395     return;
3396   }
3397   if (file_node->download_id_ != query_id) {
3398     return;
3399   }
3400 
3401   if (size != 0) {
3402     FileView file_view(file_node);
3403     if (!file_view.is_encrypted_secure()) {
3404       file_node->set_size(size);
3405     }
3406   }
3407   file_node->set_local_location(LocalFileLocation(std::move(partial_local)), ready_size, -1, -1 /* TODO */);
3408   try_flush_node(file_node, "on_partial_download");
3409 }
3410 
on_hash(QueryId query_id,string hash)3411 void FileManager::on_hash(QueryId query_id, string hash) {
3412   if (is_closed_) {
3413     return;
3414   }
3415 
3416   auto query = queries_container_.get(query_id);
3417   CHECK(query != nullptr);
3418 
3419   auto file_id = query->file_id_;
3420 
3421   auto file_node = get_file_node(file_id);
3422   LOG(DEBUG) << "Receive on_hash for file " << file_id;
3423   if (!file_node) {
3424     return;
3425   }
3426   if (file_node->upload_id_ != query_id) {
3427     return;
3428   }
3429 
3430   file_node->encryption_key_.set_value_hash(secure_storage::ValueHash::create(hash).move_as_ok());
3431 }
3432 
on_partial_upload(QueryId query_id,PartialRemoteFileLocation partial_remote,int64 ready_size)3433 void FileManager::on_partial_upload(QueryId query_id, PartialRemoteFileLocation partial_remote, int64 ready_size) {
3434   if (is_closed_) {
3435     return;
3436   }
3437 
3438   auto query = queries_container_.get(query_id);
3439   CHECK(query != nullptr);
3440 
3441   auto file_id = query->file_id_;
3442   auto file_node = get_file_node(file_id);
3443   LOG(DEBUG) << "Receive on_partial_upload for file " << file_id << " with " << partial_remote << " and ready size "
3444              << ready_size;
3445   if (!file_node) {
3446     return;
3447   }
3448   if (file_node->upload_id_ != query_id) {
3449     return;
3450   }
3451 
3452   file_node->set_partial_remote_location(std::move(partial_remote), ready_size);
3453   try_flush_node(file_node, "on_partial_upload");
3454 }
3455 
on_download_ok(QueryId query_id,FullLocalFileLocation local,int64 size,bool is_new)3456 void FileManager::on_download_ok(QueryId query_id, FullLocalFileLocation local, int64 size, bool is_new) {
3457   if (is_closed_) {
3458     return;
3459   }
3460 
3461   Query query;
3462   bool was_active;
3463   std::tie(query, was_active) = finish_query(query_id);
3464   auto file_id = query.file_id_;
3465   LOG(INFO) << "ON DOWNLOAD OK of " << (is_new ? "new" : "checked") << " file " << file_id << " of size " << size;
3466   auto r_new_file_id = register_local(std::move(local), DialogId(), size, false, false, true);
3467   Status status = Status::OK();
3468   if (r_new_file_id.is_error()) {
3469     status = Status::Error(PSLICE() << "Can't register local file after download: " << r_new_file_id.error().message());
3470   } else {
3471     if (is_new) {
3472       context_->on_new_file(size, get_file_view(r_new_file_id.ok()).get_allocated_local_size(), 1);
3473     }
3474     auto r_file_id = merge(r_new_file_id.ok(), file_id);
3475     if (r_file_id.is_error()) {
3476       status = r_file_id.move_as_error();
3477     }
3478   }
3479   if (status.is_error()) {
3480     LOG(ERROR) << status.message();
3481     return on_error_impl(get_file_node(file_id), query.type_, was_active, std::move(status));
3482   }
3483 }
3484 
on_upload_ok(QueryId query_id,FileType file_type,PartialRemoteFileLocation partial_remote,int64 size)3485 void FileManager::on_upload_ok(QueryId query_id, FileType file_type, PartialRemoteFileLocation partial_remote,
3486                                int64 size) {
3487   if (is_closed_) {
3488     return;
3489   }
3490 
3491   CHECK(partial_remote.ready_part_count_ == partial_remote.part_count_);
3492   auto some_file_id = finish_query(query_id).first.file_id_;
3493   LOG(INFO) << "ON UPLOAD OK file " << some_file_id << " of size " << size;
3494 
3495   auto file_node = get_file_node(some_file_id);
3496   if (!file_node) {
3497     return;
3498   }
3499 
3500   FileId file_id;
3501   uint64 file_id_upload_order{std::numeric_limits<uint64>::max()};
3502   for (auto id : file_node->file_ids_) {
3503     auto *info = get_file_id_info(id);
3504     if (info->upload_priority_ != 0 && info->upload_order_ < file_id_upload_order) {
3505       file_id = id;
3506       file_id_upload_order = info->upload_order_;
3507     }
3508   }
3509   if (!file_id.is_valid()) {
3510     return;
3511   }
3512 
3513   auto *file_info = get_file_id_info(file_id);
3514   LOG(INFO) << "Found being uploaded file " << file_id << " with priority " << file_info->upload_priority_;
3515   file_info->upload_priority_ = 0;
3516   file_info->download_priority_ = 0;
3517 
3518   FileView file_view(file_node);
3519   string file_name = get_file_name(file_type, file_view.suggested_path());
3520 
3521   if (file_view.is_encrypted_secret()) {
3522     tl_object_ptr<telegram_api::InputEncryptedFile> input_file;
3523     if (partial_remote.is_big_) {
3524       input_file = make_tl_object<telegram_api::inputEncryptedFileBigUploaded>(
3525           partial_remote.file_id_, partial_remote.part_count_, file_view.encryption_key().calc_fingerprint());
3526     } else {
3527       input_file = make_tl_object<telegram_api::inputEncryptedFileUploaded>(
3528           partial_remote.file_id_, partial_remote.part_count_, "", file_view.encryption_key().calc_fingerprint());
3529     }
3530     if (file_info->upload_callback_) {
3531       file_info->upload_callback_->on_upload_encrypted_ok(file_id, std::move(input_file));
3532       file_node->set_upload_pause(file_id);
3533       file_info->upload_callback_.reset();
3534     }
3535   } else if (file_view.is_secure()) {
3536     tl_object_ptr<telegram_api::InputSecureFile> input_file;
3537     input_file = make_tl_object<telegram_api::inputSecureFileUploaded>(
3538         partial_remote.file_id_, partial_remote.part_count_, "" /*md5*/, BufferSlice() /*file_hash*/,
3539         BufferSlice() /*encrypted_secret*/);
3540     if (file_info->upload_callback_) {
3541       file_info->upload_callback_->on_upload_secure_ok(file_id, std::move(input_file));
3542       file_node->upload_pause_ = file_id;
3543       file_info->upload_callback_.reset();
3544     }
3545   } else {
3546     tl_object_ptr<telegram_api::InputFile> input_file;
3547     if (partial_remote.is_big_) {
3548       input_file = make_tl_object<telegram_api::inputFileBig>(partial_remote.file_id_, partial_remote.part_count_,
3549                                                               std::move(file_name));
3550     } else {
3551       input_file = make_tl_object<telegram_api::inputFile>(partial_remote.file_id_, partial_remote.part_count_,
3552                                                            std::move(file_name), "");
3553     }
3554     if (file_info->upload_callback_) {
3555       file_info->upload_callback_->on_upload_ok(file_id, std::move(input_file));
3556       file_node->set_upload_pause(file_id);
3557       file_info->upload_callback_.reset();
3558     }
3559   }
3560 }
3561 
on_upload_full_ok(QueryId query_id,FullRemoteFileLocation remote)3562 void FileManager::on_upload_full_ok(QueryId query_id, FullRemoteFileLocation remote) {
3563   if (is_closed_) {
3564     return;
3565   }
3566 
3567   auto file_id = finish_query(query_id).first.file_id_;
3568   LOG(INFO) << "ON UPLOAD FULL OK for file " << file_id;
3569   auto new_file_id = register_remote(std::move(remote), FileLocationSource::FromServer, DialogId(), 0, 0, "");
3570   LOG_STATUS(merge(new_file_id, file_id));
3571 }
3572 
on_partial_generate(QueryId query_id,PartialLocalFileLocation partial_local,int32 expected_size)3573 void FileManager::on_partial_generate(QueryId query_id, PartialLocalFileLocation partial_local, int32 expected_size) {
3574   if (is_closed_) {
3575     return;
3576   }
3577 
3578   auto query = queries_container_.get(query_id);
3579   CHECK(query != nullptr);
3580 
3581   auto file_id = query->file_id_;
3582   auto file_node = get_file_node(file_id);
3583   auto bitmask = Bitmask(Bitmask::Decode{}, partial_local.ready_bitmask_);
3584   LOG(DEBUG) << "Receive on_partial_generate for file " << file_id << ": " << partial_local.path_ << " " << bitmask;
3585   if (!file_node) {
3586     return;
3587   }
3588   if (file_node->generate_id_ != query_id) {
3589     return;
3590   }
3591   auto ready_size = bitmask.get_total_size(partial_local.part_size_, file_node->size_);
3592   file_node->set_local_location(LocalFileLocation(partial_local), ready_size, -1, -1 /* TODO */);
3593   // TODO check for size and local_size, abort generation if needed
3594   if (expected_size > 0) {
3595     file_node->set_expected_size(expected_size);
3596   }
3597   if (!file_node->generate_was_update_) {
3598     file_node->generate_was_update_ = true;
3599     run_upload(file_node, {});
3600   }
3601   if (file_node->upload_id_ != 0) {
3602     send_closure(file_load_manager_, &FileLoadManager::update_local_file_location, file_node->upload_id_,
3603                  LocalFileLocation(std::move(partial_local)));
3604   }
3605 
3606   try_flush_node(file_node, "on_partial_generate");
3607 }
3608 
on_generate_ok(QueryId query_id,FullLocalFileLocation local)3609 void FileManager::on_generate_ok(QueryId query_id, FullLocalFileLocation local) {
3610   if (is_closed_) {
3611     return;
3612   }
3613 
3614   Query query;
3615   bool was_active;
3616   std::tie(query, was_active) = finish_query(query_id);
3617   auto generate_file_id = query.file_id_;
3618 
3619   LOG(INFO) << "Receive on_generate_ok for file " << generate_file_id << ": " << local;
3620   auto file_node = get_file_node(generate_file_id);
3621   if (!file_node) {
3622     return;
3623   }
3624 
3625   auto old_upload_id = file_node->upload_id_;
3626 
3627   auto r_new_file_id = register_local(local, DialogId(), 0);
3628   Status status;
3629   if (r_new_file_id.is_error()) {
3630     status = Status::Error(PSLICE() << "Can't register local file after generate: " << r_new_file_id.error());
3631   } else {
3632     auto result = merge(r_new_file_id.ok(), generate_file_id);
3633     if (result.is_error()) {
3634       status = result.move_as_error();
3635     }
3636   }
3637   file_node = get_file_node(generate_file_id);
3638   if (status.is_error()) {
3639     return on_error_impl(file_node, query.type_, was_active, std::move(status));
3640   }
3641   CHECK(file_node);
3642 
3643   FileView file_view(file_node);
3644   if (!file_view.has_generate_location() || !begins_with(file_view.generate_location().conversion_, "#file_id#")) {
3645     context_->on_new_file(file_view.size(), file_view.get_allocated_local_size(), 1);
3646   }
3647 
3648   run_upload(file_node, {});
3649 
3650   if (was_active) {
3651     if (old_upload_id != 0 && old_upload_id == file_node->upload_id_) {
3652       send_closure(file_load_manager_, &FileLoadManager::update_local_file_location, file_node->upload_id_,
3653                    LocalFileLocation(std::move(local)));
3654     }
3655   }
3656 }
3657 
on_error(QueryId query_id,Status status)3658 void FileManager::on_error(QueryId query_id, Status status) {
3659   if (is_closed_) {
3660     return;
3661   }
3662 
3663   Query query;
3664   bool was_active;
3665   std::tie(query, was_active) = finish_query(query_id);
3666   auto node = get_file_node(query.file_id_);
3667   if (!node) {
3668     LOG(ERROR) << "Can't find file node for " << query.file_id_ << " " << status;
3669     return;
3670   }
3671 
3672   if (query.type_ == Query::Type::UploadByHash && !G()->close_flag()) {
3673     LOG(INFO) << "Upload By Hash failed: " << status << ", restart upload";
3674     node->get_by_hash_ = false;
3675     run_upload(node, {});
3676     return;
3677   }
3678   on_error_impl(node, query.type_, was_active, std::move(status));
3679 }
3680 
on_error_impl(FileNodePtr node,Query::Type type,bool was_active,Status status)3681 void FileManager::on_error_impl(FileNodePtr node, Query::Type type, bool was_active, Status status) {
3682   SCOPE_EXIT {
3683     try_flush_node(node, "on_error");
3684   };
3685   if (status.code() != 1 && !G()->close_flag()) {
3686     LOG(WARNING) << "Failed to " << type << " file " << node->main_file_id_ << " of type " << FileView(node).get_type()
3687                  << ": " << status;
3688     if (status.code() == 0) {
3689       // Remove partial locations
3690       if (node->local_.type() == LocalFileLocation::Type::Partial &&
3691           !begins_with(status.message(), "FILE_UPLOAD_RESTART") &&
3692           !begins_with(status.message(), "FILE_DOWNLOAD_RESTART") &&
3693           !begins_with(status.message(), "FILE_DOWNLOAD_ID_INVALID") &&
3694           !begins_with(status.message(), "FILE_DOWNLOAD_LIMIT")) {
3695         CSlice path = node->local_.partial().path_;
3696         if (begins_with(path, get_files_temp_dir(FileType::Encrypted)) ||
3697             begins_with(path, get_files_temp_dir(FileType::Video))) {
3698           LOG(INFO) << "Unlink file " << path;
3699           unlink(path).ignore();
3700           node->drop_local_location();
3701         }
3702       }
3703       node->delete_partial_remote_location();
3704       status = Status::Error(400, status.message());
3705     }
3706   }
3707 
3708   if (status.message() == "FILE_PART_INVALID") {
3709     bool has_partial_small_location = node->remote_.partial && !node->remote_.partial->is_big_;
3710     FileView file_view(node);
3711     auto expected_size = file_view.expected_size(true);
3712     bool should_be_big_location = is_file_big(file_view.get_type(), expected_size);
3713 
3714     node->delete_partial_remote_location();
3715     if (has_partial_small_location && should_be_big_location) {
3716       run_upload(node, {});
3717       return;
3718     }
3719 
3720     LOG(WARNING) << "Failed to upload file " << node->main_file_id_ << ": unexpected " << status
3721                  << ", is_small = " << has_partial_small_location << ", should_be_big = " << should_be_big_location
3722                  << ", expected size = " << expected_size;
3723   }
3724 
3725   if (begins_with(status.message(), "FILE_GENERATE_LOCATION_INVALID")) {
3726     node->set_generate_location(nullptr);
3727   }
3728 
3729   if ((status.message() == "FILE_ID_INVALID" || status.message() == "LOCATION_INVALID") &&
3730       FileView(node).may_reload_photo()) {
3731     node->need_reload_photo_ = true;
3732     run_download(node, true);
3733     return;
3734   }
3735 
3736   if (FileReferenceManager::is_file_reference_error(status)) {
3737     string file_reference;
3738     Slice prefix = "#BASE64";
3739     Slice error_message = status.message();
3740     auto pos = error_message.rfind('#');
3741     if (pos < error_message.size() && begins_with(error_message.substr(pos), prefix)) {
3742       auto r_file_reference = base64_decode(error_message.substr(pos + prefix.size()));
3743       if (r_file_reference.is_ok()) {
3744         file_reference = r_file_reference.move_as_ok();
3745       } else {
3746         LOG(ERROR) << "Can't decode file reference from error " << status << ": " << r_file_reference.error();
3747       }
3748     } else {
3749       LOG(ERROR) << "Unexpected error, file_reference will be deleted just in case " << status;
3750     }
3751     CHECK(!node->file_ids_.empty());
3752     delete_file_reference(node->file_ids_.back(), file_reference);
3753     run_download(node, true);
3754     return;
3755   }
3756 
3757   if (begins_with(status.message(), "FILE_UPLOAD_RESTART")) {
3758     if (ends_with(status.message(), "WITH_FILE_REFERENCE")) {
3759       node->upload_was_update_file_reference_ = true;
3760     }
3761     run_upload(node, {});
3762     return;
3763   }
3764   if (begins_with(status.message(), "FILE_DOWNLOAD_RESTART")) {
3765     if (ends_with(status.message(), "WITH_FILE_REFERENCE")) {
3766       node->download_was_update_file_reference_ = true;
3767       run_download(node, true);
3768       return;
3769     } else if (ends_with(status.message(), "INCREASE_PART_SIZE")) {
3770       if (try_fix_partial_local_location(node)) {
3771         run_download(node, true);
3772         return;
3773       }
3774     } else {
3775       node->can_search_locally_ = false;
3776       run_download(node, true);
3777       return;
3778     }
3779   }
3780 
3781   if (!was_active) {
3782     return;
3783   }
3784 
3785   // Stop everything on error
3786   do_cancel_generate(node);
3787   do_cancel_download(node);
3788   do_cancel_upload(node);
3789 
3790   for (auto file_id : vector<FileId>(node->file_ids_)) {
3791     auto *info = get_file_id_info(file_id);
3792     if (info->download_priority_ != 0) {
3793       info->download_priority_ = 0;
3794       if (info->download_callback_) {
3795         info->download_callback_->on_download_error(file_id, status.clone());
3796         info->download_callback_.reset();
3797       }
3798     }
3799     if (info->upload_priority_ != 0) {
3800       info->upload_priority_ = 0;
3801       if (info->upload_callback_) {
3802         info->upload_callback_->on_upload_error(file_id, status.clone());
3803         info->upload_callback_.reset();
3804       }
3805     }
3806   }
3807 }
3808 
finish_query(QueryId query_id)3809 std::pair<FileManager::Query, bool> FileManager::finish_query(QueryId query_id) {
3810   SCOPE_EXIT {
3811     queries_container_.erase(query_id);
3812   };
3813   auto query = queries_container_.get(query_id);
3814   CHECK(query != nullptr);
3815 
3816   auto res = *query;
3817   auto node = get_file_node(res.file_id_);
3818   if (!node) {
3819     return std::make_pair(res, false);
3820   }
3821   bool was_active = false;
3822   if (node->generate_id_ == query_id) {
3823     node->generate_id_ = 0;
3824     node->generate_was_update_ = false;
3825     node->set_generate_priority(0, 0);
3826     was_active = true;
3827   }
3828   if (node->download_id_ == query_id) {
3829     node->download_id_ = 0;
3830     node->download_was_update_file_reference_ = false;
3831     node->is_download_started_ = false;
3832     node->set_download_priority(0);
3833     was_active = true;
3834   }
3835   if (node->upload_id_ == query_id) {
3836     node->upload_id_ = 0;
3837     node->upload_was_update_file_reference_ = false;
3838     node->set_upload_priority(0);
3839     was_active = true;
3840   }
3841   return std::make_pair(res, was_active);
3842 }
3843 
get_remote(int32 key)3844 FullRemoteFileLocation *FileManager::get_remote(int32 key) {
3845   if (key == 0) {
3846     return nullptr;
3847   }
3848   return &remote_location_info_.get(key).remote_;
3849 }
3850 
get_suggested_file_name(FileId file_id,const string & directory)3851 Result<string> FileManager::get_suggested_file_name(FileId file_id, const string &directory) {
3852   if (!file_id.is_valid()) {
3853     return Status::Error(400, "Invalid file identifier");
3854   }
3855   auto node = get_sync_file_node(file_id);
3856   if (!node) {
3857     return Status::Error(400, "Wrong file identifier");
3858   }
3859 
3860   return ::td::get_suggested_file_name(directory, PathView(node->suggested_path()).file_name());
3861 }
3862 
hangup()3863 void FileManager::hangup() {
3864   file_db_.reset();
3865   file_generate_manager_.reset();
3866   file_load_manager_.reset();
3867   while (!queries_container_.empty()) {
3868     auto ids = queries_container_.ids();
3869     for (auto id : ids) {
3870       on_error(id, Global::request_aborted_error());
3871     }
3872   }
3873   is_closed_ = true;
3874   stop();
3875 }
3876 
tear_down()3877 void FileManager::tear_down() {
3878   parent_.reset();
3879 }
3880 
3881 }  // namespace td
3882