1 #include "serializer.h"
2
3 #include "serializers/json.h"
4 #include "serializers/msgpack.h"
5
6 #include "indexer.h"
7
8 #include <doctest/doctest.h>
9 #include <loguru.hpp>
10
11 #include <stdexcept>
12
13 bool gTestOutputMode = false;
14
15 //// Elementary types
16
Reflect(Reader & visitor,uint8_t & value)17 void Reflect(Reader& visitor, uint8_t& value) {
18 if (!visitor.IsInt())
19 throw std::invalid_argument("uint8_t");
20 value = (uint8_t)visitor.GetInt();
21 }
Reflect(Writer & visitor,uint8_t & value)22 void Reflect(Writer& visitor, uint8_t& value) {
23 visitor.Int(value);
24 }
25
Reflect(Reader & visitor,short & value)26 void Reflect(Reader& visitor, short& value) {
27 if (!visitor.IsInt())
28 throw std::invalid_argument("short");
29 value = (short)visitor.GetInt();
30 }
Reflect(Writer & visitor,short & value)31 void Reflect(Writer& visitor, short& value) {
32 visitor.Int(value);
33 }
34
Reflect(Reader & visitor,unsigned short & value)35 void Reflect(Reader& visitor, unsigned short& value) {
36 if (!visitor.IsInt())
37 throw std::invalid_argument("unsigned short");
38 value = (unsigned short)visitor.GetInt();
39 }
Reflect(Writer & visitor,unsigned short & value)40 void Reflect(Writer& visitor, unsigned short& value) {
41 visitor.Int(value);
42 }
43
Reflect(Reader & visitor,int & value)44 void Reflect(Reader& visitor, int& value) {
45 if (!visitor.IsInt())
46 throw std::invalid_argument("int");
47 value = visitor.GetInt();
48 }
Reflect(Writer & visitor,int & value)49 void Reflect(Writer& visitor, int& value) {
50 visitor.Int(value);
51 }
52
Reflect(Reader & visitor,unsigned & value)53 void Reflect(Reader& visitor, unsigned& value) {
54 if (!visitor.IsUint64())
55 throw std::invalid_argument("unsigned");
56 value = visitor.GetUint32();
57 }
Reflect(Writer & visitor,unsigned & value)58 void Reflect(Writer& visitor, unsigned& value) {
59 visitor.Uint32(value);
60 }
61
Reflect(Reader & visitor,long & value)62 void Reflect(Reader& visitor, long& value) {
63 if (!visitor.IsInt64())
64 throw std::invalid_argument("long");
65 value = long(visitor.GetInt64());
66 }
Reflect(Writer & visitor,long & value)67 void Reflect(Writer& visitor, long& value) {
68 visitor.Int64(value);
69 }
70
Reflect(Reader & visitor,unsigned long & value)71 void Reflect(Reader& visitor, unsigned long& value) {
72 if (!visitor.IsUint64())
73 throw std::invalid_argument("unsigned long");
74 value = (unsigned long)visitor.GetUint64();
75 }
Reflect(Writer & visitor,unsigned long & value)76 void Reflect(Writer& visitor, unsigned long& value) {
77 visitor.Uint64(value);
78 }
79
Reflect(Reader & visitor,long long & value)80 void Reflect(Reader& visitor, long long& value) {
81 if (!visitor.IsInt64())
82 throw std::invalid_argument("long long");
83 value = visitor.GetInt64();
84 }
Reflect(Writer & visitor,long long & value)85 void Reflect(Writer& visitor, long long& value) {
86 visitor.Int64(value);
87 }
88
Reflect(Reader & visitor,unsigned long long & value)89 void Reflect(Reader& visitor, unsigned long long& value) {
90 if (!visitor.IsUint64())
91 throw std::invalid_argument("unsigned long long");
92 value = visitor.GetUint64();
93 }
Reflect(Writer & visitor,unsigned long long & value)94 void Reflect(Writer& visitor, unsigned long long& value) {
95 visitor.Uint64(value);
96 }
97
Reflect(Reader & visitor,double & value)98 void Reflect(Reader& visitor, double& value) {
99 if (!visitor.IsDouble())
100 throw std::invalid_argument("double");
101 value = visitor.GetDouble();
102 }
Reflect(Writer & visitor,double & value)103 void Reflect(Writer& visitor, double& value) {
104 visitor.Double(value);
105 }
106
Reflect(Reader & visitor,bool & value)107 void Reflect(Reader& visitor, bool& value) {
108 if (!visitor.IsBool())
109 throw std::invalid_argument("bool");
110 value = visitor.GetBool();
111 }
Reflect(Writer & visitor,bool & value)112 void Reflect(Writer& visitor, bool& value) {
113 visitor.Bool(value);
114 }
115
Reflect(Reader & visitor,std::string & value)116 void Reflect(Reader& visitor, std::string& value) {
117 if (!visitor.IsString())
118 throw std::invalid_argument("std::string");
119 value = visitor.GetString();
120 }
Reflect(Writer & visitor,std::string & value)121 void Reflect(Writer& visitor, std::string& value) {
122 visitor.String(value.c_str(), (rapidjson::SizeType)value.size());
123 }
124
Reflect(Reader &,std::string_view &)125 void Reflect(Reader&, std::string_view&) {
126 assert(0);
127 }
Reflect(Writer & visitor,std::string_view & data)128 void Reflect(Writer& visitor, std::string_view& data) {
129 if (data.empty())
130 visitor.String("");
131 else
132 visitor.String(&data[0], (rapidjson::SizeType)data.size());
133 }
134
Reflect(Reader & visitor,JsonNull & value)135 void Reflect(Reader& visitor, JsonNull& value) {
136 visitor.GetNull();
137 }
138
Reflect(Writer & visitor,JsonNull & value)139 void Reflect(Writer& visitor, JsonNull& value) {
140 visitor.Null();
141 }
142
143 // TODO: Move this to indexer.cc
Reflect(Reader & visitor,IndexInclude & value)144 void Reflect(Reader& visitor, IndexInclude& value) {
145 REFLECT_MEMBER_START();
146 REFLECT_MEMBER(line);
147 REFLECT_MEMBER(resolved_path);
148 REFLECT_MEMBER_END();
149 }
Reflect(Writer & visitor,IndexInclude & value)150 void Reflect(Writer& visitor, IndexInclude& value) {
151 REFLECT_MEMBER_START();
152 REFLECT_MEMBER(line);
153 if (gTestOutputMode) {
154 std::string basename = GetBaseName(value.resolved_path);
155 if (!StartsWith(value.resolved_path, "&"))
156 basename = "&" + basename;
157 REFLECT_MEMBER2("resolved_path", basename);
158 } else {
159 REFLECT_MEMBER(resolved_path);
160 }
161 REFLECT_MEMBER_END();
162 }
163
164 template <typename Def>
ReflectHoverAndComments(Reader & visitor,Def & def)165 void ReflectHoverAndComments(Reader& visitor, Def& def) {
166 ReflectMember(visitor, "hover", def.hover);
167 ReflectMember(visitor, "comments", def.comments);
168 }
169
170 template <typename Def>
ReflectHoverAndComments(Writer & visitor,Def & def)171 void ReflectHoverAndComments(Writer& visitor, Def& def) {
172 // Don't emit empty hover and comments in JSON test mode.
173 if (!gTestOutputMode || !def.hover.empty())
174 ReflectMember(visitor, "hover", def.hover);
175 if (!gTestOutputMode || !def.comments.empty())
176 ReflectMember(visitor, "comments", def.comments);
177 }
178
179 template <typename Def>
ReflectShortName(Reader & visitor,Def & def)180 void ReflectShortName(Reader& visitor, Def& def) {
181 if (gTestOutputMode) {
182 std::string short_name;
183 ReflectMember(visitor, "short_name", short_name);
184 def.short_name_offset = def.detailed_name.find(short_name);
185 assert(def.short_name_offset != std::string::npos);
186 def.short_name_size = short_name.size();
187 } else {
188 ReflectMember(visitor, "short_name_offset", def.short_name_offset);
189 ReflectMember(visitor, "short_name_size", def.short_name_size);
190 }
191 }
192
193 template <typename Def>
ReflectShortName(Writer & visitor,Def & def)194 void ReflectShortName(Writer& visitor, Def& def) {
195 if (gTestOutputMode) {
196 std::string short_name(
197 def.detailed_name.substr(def.short_name_offset, def.short_name_size));
198 ReflectMember(visitor, "short_name", short_name);
199 } else {
200 ReflectMember(visitor, "short_name_offset", def.short_name_offset);
201 ReflectMember(visitor, "short_name_size", def.short_name_size);
202 }
203 }
204
205 template <typename TVisitor>
Reflect(TVisitor & visitor,IndexType & value)206 void Reflect(TVisitor& visitor, IndexType& value) {
207 REFLECT_MEMBER_START();
208 REFLECT_MEMBER2("id", value.id);
209 REFLECT_MEMBER2("usr", value.usr);
210 REFLECT_MEMBER2("detailed_name", value.def.detailed_name);
211 ReflectShortName(visitor, value.def);
212 REFLECT_MEMBER2("kind", value.def.kind);
213 ReflectHoverAndComments(visitor, value.def);
214 REFLECT_MEMBER2("declarations", value.declarations);
215 REFLECT_MEMBER2("spell", value.def.spell);
216 REFLECT_MEMBER2("extent", value.def.extent);
217 REFLECT_MEMBER2("alias_of", value.def.alias_of);
218 REFLECT_MEMBER2("bases", value.def.bases);
219 REFLECT_MEMBER2("derived", value.derived);
220 REFLECT_MEMBER2("types", value.def.types);
221 REFLECT_MEMBER2("funcs", value.def.funcs);
222 REFLECT_MEMBER2("vars", value.def.vars);
223 REFLECT_MEMBER2("instances", value.instances);
224 REFLECT_MEMBER2("uses", value.uses);
225 REFLECT_MEMBER_END();
226 }
227
228 template <typename TVisitor>
Reflect(TVisitor & visitor,IndexFunc & value)229 void Reflect(TVisitor& visitor, IndexFunc& value) {
230 REFLECT_MEMBER_START();
231 REFLECT_MEMBER2("id", value.id);
232 REFLECT_MEMBER2("usr", value.usr);
233 REFLECT_MEMBER2("detailed_name", value.def.detailed_name);
234 ReflectShortName(visitor, value.def);
235 REFLECT_MEMBER2("kind", value.def.kind);
236 REFLECT_MEMBER2("storage", value.def.storage);
237 ReflectHoverAndComments(visitor, value.def);
238 REFLECT_MEMBER2("declarations", value.declarations);
239 REFLECT_MEMBER2("spell", value.def.spell);
240 REFLECT_MEMBER2("extent", value.def.extent);
241 REFLECT_MEMBER2("declaring_type", value.def.declaring_type);
242 REFLECT_MEMBER2("bases", value.def.bases);
243 REFLECT_MEMBER2("derived", value.derived);
244 REFLECT_MEMBER2("vars", value.def.vars);
245 REFLECT_MEMBER2("uses", value.uses);
246 REFLECT_MEMBER2("callees", value.def.callees);
247 REFLECT_MEMBER_END();
248 }
249
250 template <typename TVisitor>
Reflect(TVisitor & visitor,IndexVar & value)251 void Reflect(TVisitor& visitor, IndexVar& value) {
252 REFLECT_MEMBER_START();
253 REFLECT_MEMBER2("id", value.id);
254 REFLECT_MEMBER2("usr", value.usr);
255 REFLECT_MEMBER2("detailed_name", value.def.detailed_name);
256 ReflectShortName(visitor, value.def);
257 ReflectHoverAndComments(visitor, value.def);
258 REFLECT_MEMBER2("declarations", value.declarations);
259 REFLECT_MEMBER2("spell", value.def.spell);
260 REFLECT_MEMBER2("extent", value.def.extent);
261 REFLECT_MEMBER2("type", value.def.type);
262 REFLECT_MEMBER2("uses", value.uses);
263 REFLECT_MEMBER2("kind", value.def.kind);
264 REFLECT_MEMBER2("storage", value.def.storage);
265 REFLECT_MEMBER_END();
266 }
267
268 // IndexFile
ReflectMemberStart(Writer & visitor,IndexFile & value)269 bool ReflectMemberStart(Writer& visitor, IndexFile& value) {
270 // FIXME
271 auto it = value.id_cache.usr_to_type_id.find(HashUsr(""));
272 if (it != value.id_cache.usr_to_type_id.end()) {
273 value.Resolve(it->second)->def.detailed_name = "<fundamental>";
274 assert(value.Resolve(it->second)->uses.size() == 0);
275 }
276
277 DefaultReflectMemberStart(visitor);
278 return true;
279 }
280 template <typename TVisitor>
Reflect(TVisitor & visitor,IndexFile & value)281 void Reflect(TVisitor& visitor, IndexFile& value) {
282 REFLECT_MEMBER_START();
283 if (!gTestOutputMode) {
284 REFLECT_MEMBER(last_modification_time);
285 REFLECT_MEMBER(language);
286 REFLECT_MEMBER(import_file);
287 REFLECT_MEMBER(args);
288 }
289 REFLECT_MEMBER(includes);
290 if (!gTestOutputMode)
291 REFLECT_MEMBER(dependencies);
292 REFLECT_MEMBER(skipped_by_preprocessor);
293 REFLECT_MEMBER(types);
294 REFLECT_MEMBER(funcs);
295 REFLECT_MEMBER(vars);
296 REFLECT_MEMBER_END();
297 }
298
Reflect(Reader & visitor,SerializeFormat & value)299 void Reflect(Reader& visitor, SerializeFormat& value) {
300 std::string fmt = visitor.GetString();
301 value = fmt[0] == 'm' ? SerializeFormat::MessagePack : SerializeFormat::Json;
302 }
303
Reflect(Writer & visitor,SerializeFormat & value)304 void Reflect(Writer& visitor, SerializeFormat& value) {
305 switch (value) {
306 case SerializeFormat::Json:
307 visitor.String("json");
308 break;
309 case SerializeFormat::MessagePack:
310 visitor.String("msgpack");
311 break;
312 }
313 }
314
Serialize(SerializeFormat format,IndexFile & file)315 std::string Serialize(SerializeFormat format, IndexFile& file) {
316 switch (format) {
317 case SerializeFormat::Json: {
318 rapidjson::StringBuffer output;
319 rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(output);
320 writer.SetFormatOptions(
321 rapidjson::PrettyFormatOptions::kFormatSingleLineArray);
322 writer.SetIndent(' ', 2);
323 JsonWriter json_writer(&writer);
324 if (!gTestOutputMode) {
325 std::string version = std::to_string(IndexFile::kMajorVersion);
326 for (char c : version)
327 output.Put(c);
328 output.Put('\n');
329 }
330 Reflect(json_writer, file);
331 return output.GetString();
332 }
333 case SerializeFormat::MessagePack: {
334 msgpack::sbuffer buf;
335 msgpack::packer<msgpack::sbuffer> pk(&buf);
336 MessagePackWriter msgpack_writer(&pk);
337 uint64_t magic = IndexFile::kMajorVersion;
338 int version = IndexFile::kMinorVersion;
339 Reflect(msgpack_writer, magic);
340 Reflect(msgpack_writer, version);
341 Reflect(msgpack_writer, file);
342 return std::string(buf.data(), buf.size());
343 }
344 }
345 return "";
346 }
347
Deserialize(SerializeFormat format,const AbsolutePath & path,const std::string & serialized_index_content,const std::string & file_content,optional<int> expected_version)348 std::unique_ptr<IndexFile> Deserialize(
349 SerializeFormat format,
350 const AbsolutePath& path,
351 const std::string& serialized_index_content,
352 const std::string& file_content,
353 optional<int> expected_version) {
354 if (serialized_index_content.empty())
355 return nullptr;
356
357 std::unique_ptr<IndexFile> file;
358 switch (format) {
359 case SerializeFormat::Json: {
360 rapidjson::Document reader;
361 if (gTestOutputMode || !expected_version) {
362 reader.Parse(serialized_index_content.c_str());
363 } else {
364 const char* p = strchr(serialized_index_content.c_str(), '\n');
365 if (!p)
366 return nullptr;
367 if (atoi(serialized_index_content.c_str()) != *expected_version)
368 return nullptr;
369 reader.Parse(p + 1);
370 }
371 if (reader.HasParseError())
372 return nullptr;
373
374 file = std::make_unique<IndexFile>(path, file_content);
375 JsonReader json_reader{&reader};
376 try {
377 Reflect(json_reader, *file);
378 } catch (std::invalid_argument& e) {
379 LOG_S(INFO) << "'" << path << "': failed to deserialize "
380 << json_reader.GetPath() << "." << e.what();
381 return nullptr;
382 }
383 break;
384 }
385
386 case SerializeFormat::MessagePack: {
387 try {
388 int major, minor;
389 if (serialized_index_content.size() < 8)
390 throw std::invalid_argument("Invalid");
391 msgpack::unpacker upk;
392 upk.reserve_buffer(serialized_index_content.size());
393 memcpy(upk.buffer(), serialized_index_content.data(),
394 serialized_index_content.size());
395 upk.buffer_consumed(serialized_index_content.size());
396 file = std::make_unique<IndexFile>(path, file_content);
397 MessagePackReader reader(&upk);
398 Reflect(reader, major);
399 Reflect(reader, minor);
400 if (major != IndexFile::kMajorVersion ||
401 minor != IndexFile::kMinorVersion)
402 throw std::invalid_argument("Invalid version");
403 Reflect(reader, *file);
404 } catch (std::invalid_argument& e) {
405 LOG_S(INFO) << "Failed to deserialize msgpack '" << path
406 << "': " << e.what();
407 return nullptr;
408 }
409 break;
410 }
411 }
412
413 // Restore non-serialized state.
414 file->path = path;
415 file->id_cache.primary_file = file->path;
416 for (const auto& type : file->types) {
417 file->id_cache.type_id_to_usr[type.id] = type.usr;
418 file->id_cache.usr_to_type_id[type.usr] = type.id;
419 }
420 for (const auto& func : file->funcs) {
421 file->id_cache.func_id_to_usr[func.id] = func.usr;
422 file->id_cache.usr_to_func_id[func.usr] = func.id;
423 }
424 for (const auto& var : file->vars) {
425 file->id_cache.var_id_to_usr[var.id] = var.usr;
426 file->id_cache.usr_to_var_id[var.usr] = var.id;
427 }
428
429 return file;
430 }
431
SetTestOutputMode()432 void SetTestOutputMode() {
433 gTestOutputMode = true;
434 }
435
436 TEST_SUITE("Serializer utils") {
437 TEST_CASE("GetBaseName") {
438 REQUIRE(GetBaseName("foo.cc") == "foo.cc");
439 REQUIRE(GetBaseName("foo/foo.cc") == "foo.cc");
440 REQUIRE(GetBaseName("/foo.cc") == "foo.cc");
441 REQUIRE(GetBaseName("///foo.cc") == "foo.cc");
442 REQUIRE(GetBaseName("bar/") == "bar/");
443 REQUIRE(GetBaseName("foobar/bar/") ==
444 "foobar/bar/"); // TODO: Should be bar, but good enough.
445 }
446 }
447