1 //
2 // Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2021
3 //
4 // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 //
7 #include "td/telegram/FileReferenceManager.h"
8 
9 #include "td/telegram/AnimationsManager.h"
10 #include "td/telegram/BackgroundManager.h"
11 #include "td/telegram/ConfigManager.h"
12 #include "td/telegram/ContactsManager.h"
13 #include "td/telegram/files/FileManager.h"
14 #include "td/telegram/Global.h"
15 #include "td/telegram/MessagesManager.h"
16 #include "td/telegram/StickerSetId.h"
17 #include "td/telegram/StickersManager.h"
18 #include "td/telegram/Td.h"
19 #include "td/telegram/WebPageId.h"
20 #include "td/telegram/WebPagesManager.h"
21 
22 #include "td/utils/algorithm.h"
23 #include "td/utils/common.h"
24 #include "td/utils/logging.h"
25 #include "td/utils/misc.h"
26 #include "td/utils/overloaded.h"
27 #include "td/utils/SliceBuilder.h"
28 #include "td/utils/Time.h"
29 
30 namespace td {
31 
32 int VERBOSITY_NAME(file_references) = VERBOSITY_NAME(INFO);
33 
is_file_reference_error(const Status & error)34 bool FileReferenceManager::is_file_reference_error(const Status &error) {
35   return error.is_error() && error.code() == 400 && begins_with(error.message(), "FILE_REFERENCE_");
36 }
37 
get_file_reference_error_pos(const Status & error)38 size_t FileReferenceManager::get_file_reference_error_pos(const Status &error) {
39   if (!is_file_reference_error(error)) {
40     return 0;
41   }
42   auto offset = Slice("FILE_REFERENCE_").size();
43   if (error.message().size() <= offset || !is_digit(error.message()[offset])) {
44     return 0;
45   }
46   return to_integer<size_t>(error.message().substr(offset)) + 1;
47 }
48 
49 /*
50 fileSourceMessage chat_id:int53 message_id:int53 = FileSource;           // repaired with get_message_from_server
51 fileSourceUserProfilePhoto user_id:int32 photo_id:int64 = FileSource;    // repaired with photos.getUserPhotos
52 fileSourceBasicGroupPhoto basic_group_id:int32 = FileSource;             // no need to repair
53 fileSourceSupergroupPhoto supergroup_id:int32 = FileSource;              // no need to repair
54 fileSourceWebPage url:string = FileSource;                               // repaired with messages.getWebPage
55 fileSourceWallpapers = FileSource;                                       // can't be repaired
56 fileSourceSavedAnimations = FileSource;                                  // repaired with messages.getSavedGifs
57 fileSourceRecentStickers is_attached:Bool = FileSource;                  // repaired with messages.getRecentStickers, not reliable
58 fileSourceFavoriteStickers = FileSource;                                 // repaired with messages.getFavedStickers, not reliable
59 fileSourceBackground background_id:int64 access_hash:int64 = FileSource; // repaired with account.getWallPaper
60 fileSourceBasicGroupFull basic_group_id:int32 = FileSource;              // repaired with messages.getFullChat
61 fileSourceSupergroupFull supergroup_id:int32 = FileSource;               // repaired with messages.getFullChannel
62 fileSourceAppConfig = FileSource;                                        // repaired with help.getAppConfig, not reliable
63 */
64 
get_current_file_source_id() const65 FileSourceId FileReferenceManager::get_current_file_source_id() const {
66   return FileSourceId(narrow_cast<int32>(file_sources_.size()));
67 }
68 
69 template <class T>
add_file_source_id(T source,Slice source_str)70 FileSourceId FileReferenceManager::add_file_source_id(T source, Slice source_str) {
71   file_sources_.emplace_back(std::move(source));
72   VLOG(file_references) << "Create file source " << file_sources_.size() << " for " << source_str;
73   return get_current_file_source_id();
74 }
75 
create_message_file_source(FullMessageId full_message_id)76 FileSourceId FileReferenceManager::create_message_file_source(FullMessageId full_message_id) {
77   FileSourceMessage source{full_message_id};
78   return add_file_source_id(source, PSLICE() << full_message_id);
79 }
80 
create_user_photo_file_source(UserId user_id,int64 photo_id)81 FileSourceId FileReferenceManager::create_user_photo_file_source(UserId user_id, int64 photo_id) {
82   FileSourceUserPhoto source{photo_id, user_id};
83   return add_file_source_id(source, PSLICE() << "photo " << photo_id << " of " << user_id);
84 }
85 
create_web_page_file_source(string url)86 FileSourceId FileReferenceManager::create_web_page_file_source(string url) {
87   FileSourceWebPage source{std::move(url)};
88   auto source_str = PSTRING() << "web page of " << source.url;
89   return add_file_source_id(std::move(source), source_str);
90 }
91 
create_saved_animations_file_source()92 FileSourceId FileReferenceManager::create_saved_animations_file_source() {
93   FileSourceSavedAnimations source;
94   return add_file_source_id(source, "saved animations");
95 }
96 
create_recent_stickers_file_source(bool is_attached)97 FileSourceId FileReferenceManager::create_recent_stickers_file_source(bool is_attached) {
98   FileSourceRecentStickers source{is_attached};
99   return add_file_source_id(source, PSLICE() << "recent " << (is_attached ? "attached " : "") << "stickers");
100 }
101 
create_favorite_stickers_file_source()102 FileSourceId FileReferenceManager::create_favorite_stickers_file_source() {
103   FileSourceFavoriteStickers source;
104   return add_file_source_id(source, "favorite stickers");
105 }
106 
create_background_file_source(BackgroundId background_id,int64 access_hash)107 FileSourceId FileReferenceManager::create_background_file_source(BackgroundId background_id, int64 access_hash) {
108   FileSourceBackground source{background_id, access_hash};
109   return add_file_source_id(source, PSLICE() << background_id);
110 }
111 
create_chat_full_file_source(ChatId chat_id)112 FileSourceId FileReferenceManager::create_chat_full_file_source(ChatId chat_id) {
113   FileSourceChatFull source{chat_id};
114   return add_file_source_id(source, PSLICE() << "full " << chat_id);
115 }
116 
create_channel_full_file_source(ChannelId channel_id)117 FileSourceId FileReferenceManager::create_channel_full_file_source(ChannelId channel_id) {
118   FileSourceChannelFull source{channel_id};
119   return add_file_source_id(source, PSLICE() << "full " << channel_id);
120 }
121 
create_app_config_file_source()122 FileSourceId FileReferenceManager::create_app_config_file_source() {
123   FileSourceAppConfig source;
124   return add_file_source_id(source, "app config");
125 }
126 
add_file_source(NodeId node_id,FileSourceId file_source_id)127 bool FileReferenceManager::add_file_source(NodeId node_id, FileSourceId file_source_id) {
128   bool is_added = nodes_[node_id].file_source_ids.add(file_source_id);
129   VLOG(file_references) << "Add " << (is_added ? "new" : "old") << ' ' << file_source_id << " for file " << node_id;
130   return is_added;
131 }
132 
remove_file_source(NodeId node_id,FileSourceId file_source_id)133 bool FileReferenceManager::remove_file_source(NodeId node_id, FileSourceId file_source_id) {
134   bool is_removed = nodes_[node_id].file_source_ids.remove(file_source_id);
135   if (is_removed) {
136     VLOG(file_references) << "Remove " << file_source_id << " from file " << node_id;
137   } else {
138     VLOG(file_references) << "Can't find " << file_source_id << " from file " << node_id << " to remove it";
139   }
140   return is_removed;
141 }
142 
get_some_file_sources(NodeId node_id)143 vector<FileSourceId> FileReferenceManager::get_some_file_sources(NodeId node_id) {
144   auto it = nodes_.find(node_id);
145   if (it == nodes_.end()) {
146     return {};
147   }
148   return it->second.file_source_ids.get_some_elements();
149 }
150 
get_some_message_file_sources(NodeId node_id)151 vector<FullMessageId> FileReferenceManager::get_some_message_file_sources(NodeId node_id) {
152   auto file_source_ids = get_some_file_sources(node_id);
153 
154   vector<FullMessageId> result;
155   for (auto file_source_id : file_source_ids) {
156     auto index = static_cast<size_t>(file_source_id.get()) - 1;
157     CHECK(index < file_sources_.size());
158     const auto &file_source = file_sources_[index];
159     if (file_source.get_offset() == 0) {
160       result.push_back(file_source.get<FileSourceMessage>().full_message_id);
161     }
162   }
163   return result;
164 }
165 
merge(NodeId to_node_id,NodeId from_node_id)166 void FileReferenceManager::merge(NodeId to_node_id, NodeId from_node_id) {
167   auto from_it = nodes_.find(from_node_id);
168   if (from_it == nodes_.end()) {
169     return;
170   }
171 
172   auto &to = nodes_[to_node_id];
173   auto &from = from_it->second;
174   VLOG(file_references) << "Merge " << to.file_source_ids.size() << " and " << from.file_source_ids.size()
175                         << " sources of files " << to_node_id << " and " << from_node_id;
176   CHECK(!to.query || to.query->proxy.is_empty());
177   CHECK(!from.query || from.query->proxy.is_empty());
178   if (to.query || from.query) {
179     if (!to.query) {
180       to.query = make_unique<Query>();
181       to.query->generation = ++query_generation_;
182     }
183     if (from.query) {
184       combine(to.query->promises, std::move(from.query->promises));
185       to.query->active_queries += from.query->active_queries;
186       from.query->proxy = Destination(to_node_id, to.query->generation);
187     }
188   }
189   to.file_source_ids.merge(std::move(from.file_source_ids));
190   run_node(to_node_id);
191   run_node(from_node_id);
192 }
193 
run_node(NodeId node_id)194 void FileReferenceManager::run_node(NodeId node_id) {
195   Node &node = nodes_[node_id];
196   if (!node.query) {
197     return;
198   }
199   if (node.query->active_queries != 0) {
200     return;
201   }
202   VLOG(file_references) << "Trying to repair file reference for file " << node_id;
203   if (node.query->promises.empty()) {
204     node.query = {};
205     return;
206   }
207   if (!node.file_source_ids.has_next()) {
208     VLOG(file_references) << "Have no more file sources to repair file reference for file " << node_id;
209     for (auto &p : node.query->promises) {
210       if (node.file_source_ids.empty()) {
211         p.set_error(Status::Error(400, "File source is not found"));
212       } else {
213         p.set_error(Status::Error(429, "Too Many Requests: retry after 1"));
214       }
215     }
216     node.query = {};
217     return;
218   }
219   if (node.last_successful_repair_time >= Time::now() - 60) {
220     VLOG(file_references) << "Recently repaired file reference for file " << node_id << ", do not try again";
221     for (auto &p : node.query->promises) {
222       p.set_error(Status::Error(429, "Too Many Requests: retry after 60"));
223     }
224     node.query = {};
225     return;
226   }
227   auto file_source_id = node.file_source_ids.next();
228   send_query(Destination(node_id, node.query->generation), file_source_id);
229 }
230 
send_query(Destination dest,FileSourceId file_source_id)231 void FileReferenceManager::send_query(Destination dest, FileSourceId file_source_id) {
232   VLOG(file_references) << "Send file reference repair query for file " << dest.node_id << " with generation "
233                         << dest.generation << " from " << file_source_id;
234   auto &node = nodes_[dest.node_id];
235   node.query->active_queries++;
236 
237   auto promise = PromiseCreator::lambda([dest, file_source_id, actor_id = actor_id(this),
238                                          file_manager_actor_id = G()->file_manager()](Result<Unit> result) {
239     auto new_promise = PromiseCreator::lambda([dest, file_source_id, actor_id](Result<Unit> result) {
240       Status status;
241       if (result.is_error()) {
242         status = result.move_as_error();
243       }
244       send_closure(actor_id, &FileReferenceManager::on_query_result, dest, file_source_id, std::move(status), 0);
245     });
246 
247     send_closure(file_manager_actor_id, &FileManager::on_file_reference_repaired, dest.node_id, file_source_id,
248                  std::move(result), std::move(new_promise));
249   });
250   auto index = static_cast<size_t>(file_source_id.get()) - 1;
251   CHECK(index < file_sources_.size());
252   file_sources_[index].visit(overloaded(
253       [&](const FileSourceMessage &source) {
254         send_closure_later(G()->messages_manager(), &MessagesManager::get_message_from_server, source.full_message_id,
255                            std::move(promise), "FileSourceMessage", nullptr);
256       },
257       [&](const FileSourceUserPhoto &source) {
258         send_closure_later(G()->contacts_manager(), &ContactsManager::reload_user_profile_photo, source.user_id,
259                            source.photo_id, std::move(promise));
260       },
261       [&](const FileSourceChatPhoto &source) {
262         send_closure_later(G()->contacts_manager(), &ContactsManager::reload_chat, source.chat_id, std::move(promise));
263       },
264       [&](const FileSourceChannelPhoto &source) {
265         send_closure_later(G()->contacts_manager(), &ContactsManager::reload_channel, source.channel_id,
266                            std::move(promise));
267       },
268       [&](const FileSourceWallpapers &source) { promise.set_error(Status::Error("Can't repair old wallpapers")); },
269       [&](const FileSourceWebPage &source) {
270         send_closure_later(G()->web_pages_manager(), &WebPagesManager::reload_web_page_by_url, source.url,
271                            PromiseCreator::lambda([promise = std::move(promise)](Result<WebPageId> &&result) mutable {
272                              if (result.is_error()) {
273                                promise.set_error(result.move_as_error());
274                              } else {
275                                promise.set_value(Unit());
276                              }
277                            }));
278       },
279       [&](const FileSourceSavedAnimations &source) {
280         send_closure_later(G()->animations_manager(), &AnimationsManager::repair_saved_animations, std::move(promise));
281       },
282       [&](const FileSourceRecentStickers &source) {
283         send_closure_later(G()->stickers_manager(), &StickersManager::repair_recent_stickers, source.is_attached,
284                            std::move(promise));
285       },
286       [&](const FileSourceFavoriteStickers &source) {
287         send_closure_later(G()->stickers_manager(), &StickersManager::repair_favorite_stickers, std::move(promise));
288       },
289       [&](const FileSourceBackground &source) {
290         send_closure_later(G()->background_manager(), &BackgroundManager::reload_background, source.background_id,
291                            source.access_hash, std::move(promise));
292       },
293       [&](const FileSourceChatFull &source) {
294         send_closure_later(G()->contacts_manager(), &ContactsManager::reload_chat_full, source.chat_id,
295                            std::move(promise));
296       },
297       [&](const FileSourceChannelFull &source) {
298         send_closure_later(G()->contacts_manager(), &ContactsManager::reload_channel_full, source.channel_id,
299                            std::move(promise), "repair file reference");
300       },
301       [&](const FileSourceAppConfig &source) {
302         send_closure_later(G()->config_manager(), &ConfigManager::reget_app_config, std::move(promise));
303       }));
304 }
305 
on_query_result(Destination dest,FileSourceId file_source_id,Status status,int32 sub)306 FileReferenceManager::Destination FileReferenceManager::on_query_result(Destination dest, FileSourceId file_source_id,
307                                                                         Status status, int32 sub) {
308   if (G()->close_flag()) {
309     VLOG(file_references) << "Ignore file reference repair from " << file_source_id << " during closing";
310     return dest;
311   }
312 
313   VLOG(file_references) << "Receive result of file reference repair query for file " << dest.node_id
314                         << " with generation " << dest.generation << " from " << file_source_id << ": " << status << " "
315                         << sub;
316   auto &node = nodes_[dest.node_id];
317 
318   auto query = node.query.get();
319   if (!query) {
320     return dest;
321   }
322   if (query->generation != dest.generation) {
323     return dest;
324   }
325   query->active_queries--;
326   CHECK(query->active_queries >= 0);
327 
328   if (!query->proxy.is_empty()) {
329     query->active_queries -= sub;
330     CHECK(query->active_queries >= 0);
331     auto new_proxy = on_query_result(query->proxy, file_source_id, std::move(status), query->active_queries);
332     query->proxy = new_proxy;
333     run_node(dest.node_id);
334     return new_proxy;
335   }
336 
337   if (status.is_ok()) {
338     node.last_successful_repair_time = Time::now();
339     for (auto &p : query->promises) {
340       p.set_value(Unit());
341     }
342     node.query = {};
343   }
344 
345   run_node(dest.node_id);
346   return dest;
347 }
348 
repair_file_reference(NodeId node_id,Promise<> promise)349 void FileReferenceManager::repair_file_reference(NodeId node_id, Promise<> promise) {
350   auto main_file_id = G()->td().get_actor_unsafe()->file_manager_->get_file_view(node_id).file_id();
351   VLOG(file_references) << "Repair file reference for file " << node_id << "/" << main_file_id;
352   node_id = main_file_id;
353   auto &node = nodes_[node_id];
354   if (!node.query) {
355     node.query = make_unique<Query>();
356     node.query->generation = ++query_generation_;
357     node.file_source_ids.reset_position();
358     VLOG(file_references) << "Create new file reference repair query with generation " << query_generation_;
359   }
360   node.query->promises.push_back(std::move(promise));
361   run_node(node_id);
362 }
363 
reload_photo(PhotoSizeSource source,Promise<Unit> promise)364 void FileReferenceManager::reload_photo(PhotoSizeSource source, Promise<Unit> promise) {
365   switch (source.get_type("reload_photo")) {
366     case PhotoSizeSource::Type::DialogPhotoBig:
367     case PhotoSizeSource::Type::DialogPhotoSmall:
368     case PhotoSizeSource::Type::DialogPhotoBigLegacy:
369     case PhotoSizeSource::Type::DialogPhotoSmallLegacy:
370       send_closure(G()->contacts_manager(), &ContactsManager::reload_dialog_info, source.dialog_photo().dialog_id,
371                    std::move(promise));
372       break;
373     case PhotoSizeSource::Type::StickerSetThumbnail:
374     case PhotoSizeSource::Type::StickerSetThumbnailLegacy:
375     case PhotoSizeSource::Type::StickerSetThumbnailVersion:
376       send_closure(G()->stickers_manager(), &StickersManager::reload_sticker_set,
377                    StickerSetId(source.sticker_set_thumbnail().sticker_set_id),
378                    source.sticker_set_thumbnail().sticker_set_access_hash, std::move(promise));
379       break;
380     case PhotoSizeSource::Type::Legacy:
381     case PhotoSizeSource::Type::FullLegacy:
382     case PhotoSizeSource::Type::Thumbnail:
383       promise.set_error(Status::Error("Unexpected PhotoSizeSource type"));
384       break;
385     default:
386       UNREACHABLE();
387   }
388 }
389 
390 }  // namespace td
391