1 //===-- YAMLSerialization.cpp ------------------------------------*- C++-*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // A YAML index file is a sequence of tagged entries. 10 // Each entry either encodes a Symbol or the list of references to a symbol 11 // (a "ref bundle"). 12 // 13 //===----------------------------------------------------------------------===// operator <<(llvm::raw_ostream & OS,SymbolOrigin O)14 15 #include "Index.h" 16 #include "Relation.h" 17 #include "Serialization.h" 18 #include "SymbolLocation.h" 19 #include "SymbolOrigin.h" 20 #include "dex/Dex.h" 21 #include "support/Logger.h" 22 #include "support/Trace.h" 23 #include "llvm/ADT/Optional.h" 24 #include "llvm/ADT/SmallVector.h" 25 #include "llvm/ADT/StringRef.h" 26 #include "llvm/Support/Allocator.h" 27 #include "llvm/Support/Errc.h" 28 #include "llvm/Support/MemoryBuffer.h" 29 #include "llvm/Support/StringSaver.h" 30 #include "llvm/Support/YAMLTraits.h" 31 #include "llvm/Support/raw_ostream.h" 32 #include <cstdint> 33 34 LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences) 35 LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Ref) 36 37 namespace { 38 using RefBundle = 39 std::pair<clang::clangd::SymbolID, std::vector<clang::clangd::Ref>>; 40 // This is a pale imitation of std::variant<Symbol, RefBundle, Relation> 41 struct VariantEntry { 42 llvm::Optional<clang::clangd::Symbol> Symbol; 43 llvm::Optional<RefBundle> Refs; 44 llvm::Optional<clang::clangd::Relation> Relation; 45 llvm::Optional<clang::clangd::IncludeGraphNode> Source; 46 llvm::Optional<clang::tooling::CompileCommand> Cmd; 47 }; 48 // A class helps YAML to serialize the 32-bit encoded position (Line&Column), 49 // as YAMLIO can't directly map bitfields. 50 struct YPosition { 51 uint32_t Line; 52 uint32_t Column; 53 }; 54 55 // avoid ODR violation of specialization for non-owned CompileCommand 56 struct CompileCommandYAML : clang::tooling::CompileCommand {}; 57 58 } // namespace 59 namespace llvm { 60 namespace yaml { 61 62 using clang::clangd::FileDigest; 63 using clang::clangd::IncludeGraph; 64 using clang::clangd::IncludeGraphNode; 65 using clang::clangd::Ref; 66 using clang::clangd::RefKind; 67 using clang::clangd::Relation; 68 using clang::clangd::RelationKind; 69 using clang::clangd::Symbol; 70 using clang::clangd::SymbolID; 71 using clang::clangd::SymbolLocation; 72 using clang::clangd::SymbolOrigin; 73 using clang::index::SymbolInfo; 74 using clang::index::SymbolKind; 75 using clang::index::SymbolLanguage; 76 using clang::index::SymbolRole; 77 using clang::tooling::CompileCommand; 78 79 // Helper to (de)serialize the SymbolID. We serialize it as a hex string. 80 struct NormalizedSymbolID { 81 NormalizedSymbolID(IO &) {} 82 NormalizedSymbolID(IO &, const SymbolID &ID) { 83 llvm::raw_string_ostream OS(HexString); 84 OS << ID; 85 } 86 87 SymbolID denormalize(IO &I) { 88 auto ID = SymbolID::fromStr(HexString); 89 if (!ID) { 90 I.setError(llvm::toString(ID.takeError())); 91 return SymbolID(); 92 } 93 return *ID; 94 } 95 96 std::string HexString; 97 }; 98 99 struct NormalizedSymbolFlag { 100 NormalizedSymbolFlag(IO &) {} 101 NormalizedSymbolFlag(IO &, Symbol::SymbolFlag F) { 102 Flag = static_cast<uint8_t>(F); 103 } 104 105 Symbol::SymbolFlag denormalize(IO &) { 106 return static_cast<Symbol::SymbolFlag>(Flag); 107 } 108 109 uint8_t Flag = 0; 110 }; 111 112 struct NormalizedSymbolOrigin { 113 NormalizedSymbolOrigin(IO &) {} 114 NormalizedSymbolOrigin(IO &, SymbolOrigin O) { 115 Origin = static_cast<uint8_t>(O); 116 } 117 118 SymbolOrigin denormalize(IO &) { return static_cast<SymbolOrigin>(Origin); } 119 120 uint8_t Origin = 0; 121 }; 122 123 template <> struct MappingTraits<YPosition> { 124 static void mapping(IO &IO, YPosition &Value) { 125 IO.mapRequired("Line", Value.Line); 126 IO.mapRequired("Column", Value.Column); 127 } 128 }; 129 130 struct NormalizedPosition { 131 using Position = clang::clangd::SymbolLocation::Position; 132 NormalizedPosition(IO &) {} 133 NormalizedPosition(IO &, const Position &Pos) { 134 P.Line = Pos.line(); 135 P.Column = Pos.column(); 136 } 137 138 Position denormalize(IO &) { 139 Position Pos; 140 Pos.setLine(P.Line); 141 Pos.setColumn(P.Column); 142 return Pos; 143 } 144 YPosition P; 145 }; 146 147 struct NormalizedFileURI { 148 NormalizedFileURI(IO &) {} 149 NormalizedFileURI(IO &, const char *FileURI) { URI = FileURI; } 150 151 const char *denormalize(IO &IO) { 152 assert(IO.getContext() && 153 "Expecting an UniqueStringSaver to allocate data"); 154 return static_cast<llvm::UniqueStringSaver *>(IO.getContext()) 155 ->save(URI) 156 .data(); 157 } 158 159 std::string URI; 160 }; 161 162 template <> struct MappingTraits<SymbolLocation> { 163 static void mapping(IO &IO, SymbolLocation &Value) { 164 MappingNormalization<NormalizedFileURI, const char *> NFile(IO, 165 Value.FileURI); 166 IO.mapRequired("FileURI", NFile->URI); 167 MappingNormalization<NormalizedPosition, SymbolLocation::Position> NStart( 168 IO, Value.Start); 169 IO.mapRequired("Start", NStart->P); 170 MappingNormalization<NormalizedPosition, SymbolLocation::Position> NEnd( 171 IO, Value.End); 172 IO.mapRequired("End", NEnd->P); 173 } 174 }; 175 176 template <> struct MappingTraits<SymbolInfo> { 177 static void mapping(IO &io, SymbolInfo &SymInfo) { 178 // FIXME: expose other fields? 179 io.mapRequired("Kind", SymInfo.Kind); 180 io.mapRequired("Lang", SymInfo.Lang); 181 } 182 }; 183 184 template <> 185 struct MappingTraits<clang::clangd::Symbol::IncludeHeaderWithReferences> { 186 static void mapping(IO &io, 187 clang::clangd::Symbol::IncludeHeaderWithReferences &Inc) { 188 io.mapRequired("Header", Inc.IncludeHeader); 189 io.mapRequired("References", Inc.References); 190 } 191 }; 192 193 template <> struct MappingTraits<Symbol> { 194 static void mapping(IO &IO, Symbol &Sym) { 195 MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, Sym.ID); 196 MappingNormalization<NormalizedSymbolFlag, Symbol::SymbolFlag> NSymbolFlag( 197 IO, Sym.Flags); 198 MappingNormalization<NormalizedSymbolOrigin, SymbolOrigin> NSymbolOrigin( 199 IO, Sym.Origin); 200 IO.mapRequired("ID", NSymbolID->HexString); 201 IO.mapRequired("Name", Sym.Name); 202 IO.mapRequired("Scope", Sym.Scope); 203 IO.mapRequired("SymInfo", Sym.SymInfo); 204 IO.mapOptional("CanonicalDeclaration", Sym.CanonicalDeclaration, 205 SymbolLocation()); 206 IO.mapOptional("Definition", Sym.Definition, SymbolLocation()); 207 IO.mapOptional("References", Sym.References, 0u); 208 IO.mapOptional("Origin", NSymbolOrigin->Origin); 209 IO.mapOptional("Flags", NSymbolFlag->Flag); 210 IO.mapOptional("Signature", Sym.Signature); 211 IO.mapOptional("TemplateSpecializationArgs", 212 Sym.TemplateSpecializationArgs); 213 IO.mapOptional("CompletionSnippetSuffix", Sym.CompletionSnippetSuffix); 214 IO.mapOptional("Documentation", Sym.Documentation); 215 IO.mapOptional("ReturnType", Sym.ReturnType); 216 IO.mapOptional("Type", Sym.Type); 217 IO.mapOptional("IncludeHeaders", Sym.IncludeHeaders); 218 } 219 }; 220 221 template <> struct ScalarEnumerationTraits<SymbolLanguage> { 222 static void enumeration(IO &IO, SymbolLanguage &Value) { 223 IO.enumCase(Value, "C", SymbolLanguage::C); 224 IO.enumCase(Value, "Cpp", SymbolLanguage::CXX); 225 IO.enumCase(Value, "ObjC", SymbolLanguage::ObjC); 226 IO.enumCase(Value, "Swift", SymbolLanguage::Swift); 227 } 228 }; 229 230 template <> struct ScalarEnumerationTraits<SymbolKind> { 231 static void enumeration(IO &IO, SymbolKind &Value) { 232 #define DEFINE_ENUM(name) IO.enumCase(Value, #name, SymbolKind::name) 233 234 DEFINE_ENUM(Unknown); 235 DEFINE_ENUM(Function); 236 DEFINE_ENUM(Module); 237 DEFINE_ENUM(Namespace); 238 DEFINE_ENUM(NamespaceAlias); 239 DEFINE_ENUM(Macro); 240 DEFINE_ENUM(Enum); 241 DEFINE_ENUM(Struct); 242 DEFINE_ENUM(Class); 243 DEFINE_ENUM(Protocol); 244 DEFINE_ENUM(Extension); 245 DEFINE_ENUM(Union); 246 DEFINE_ENUM(TypeAlias); 247 DEFINE_ENUM(Function); 248 DEFINE_ENUM(Variable); 249 DEFINE_ENUM(Field); 250 DEFINE_ENUM(EnumConstant); 251 DEFINE_ENUM(InstanceMethod); 252 DEFINE_ENUM(ClassMethod); 253 DEFINE_ENUM(StaticMethod); 254 DEFINE_ENUM(InstanceProperty); 255 DEFINE_ENUM(ClassProperty); 256 DEFINE_ENUM(StaticProperty); 257 DEFINE_ENUM(Constructor); 258 DEFINE_ENUM(Destructor); 259 DEFINE_ENUM(ConversionFunction); 260 DEFINE_ENUM(Parameter); 261 DEFINE_ENUM(Using); 262 263 #undef DEFINE_ENUM 264 } 265 }; 266 267 template <> struct MappingTraits<RefBundle> { 268 static void mapping(IO &IO, RefBundle &Refs) { 269 MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, 270 Refs.first); 271 IO.mapRequired("ID", NSymbolID->HexString); 272 IO.mapRequired("References", Refs.second); 273 } 274 }; 275 276 struct NormalizedRefKind { 277 NormalizedRefKind(IO &) {} 278 NormalizedRefKind(IO &, RefKind O) { Kind = static_cast<uint8_t>(O); } 279 280 RefKind denormalize(IO &) { return static_cast<RefKind>(Kind); } 281 282 uint8_t Kind = 0; 283 }; 284 285 template <> struct MappingTraits<Ref> { 286 static void mapping(IO &IO, Ref &R) { 287 MappingNormalization<NormalizedRefKind, RefKind> NKind(IO, R.Kind); 288 IO.mapRequired("Kind", NKind->Kind); 289 IO.mapRequired("Location", R.Location); 290 } 291 }; 292 293 struct NormalizedSymbolRole { 294 NormalizedSymbolRole(IO &) {} 295 NormalizedSymbolRole(IO &IO, RelationKind R) { 296 Kind = static_cast<uint8_t>(R); 297 } 298 299 RelationKind denormalize(IO &IO) { return static_cast<RelationKind>(Kind); } 300 301 uint8_t Kind = 0; 302 }; 303 304 template <> struct MappingTraits<SymbolID> { 305 static void mapping(IO &IO, SymbolID &ID) { 306 MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, ID); 307 IO.mapRequired("ID", NSymbolID->HexString); 308 } 309 }; 310 311 template <> struct MappingTraits<Relation> { 312 static void mapping(IO &IO, Relation &Relation) { 313 MappingNormalization<NormalizedSymbolRole, RelationKind> NRole( 314 IO, Relation.Predicate); 315 IO.mapRequired("Subject", Relation.Subject); 316 IO.mapRequired("Predicate", NRole->Kind); 317 IO.mapRequired("Object", Relation.Object); 318 } 319 }; 320 321 struct NormalizedSourceFlag { 322 NormalizedSourceFlag(IO &) {} 323 NormalizedSourceFlag(IO &, IncludeGraphNode::SourceFlag O) { 324 Flag = static_cast<uint8_t>(O); 325 } 326 327 IncludeGraphNode::SourceFlag denormalize(IO &) { 328 return static_cast<IncludeGraphNode::SourceFlag>(Flag); 329 } 330 331 uint8_t Flag = 0; 332 }; 333 334 struct NormalizedFileDigest { 335 NormalizedFileDigest(IO &) {} 336 NormalizedFileDigest(IO &, const FileDigest &Digest) { 337 HexString = llvm::toHex(Digest); 338 } 339 340 FileDigest denormalize(IO &I) { 341 FileDigest Digest; 342 if (HexString.size() == Digest.size() * 2 && 343 llvm::all_of(HexString, llvm::isHexDigit)) { 344 memcpy(Digest.data(), llvm::fromHex(HexString).data(), Digest.size()); 345 } else { 346 I.setError(std::string("Bad hex file digest: ") + HexString); 347 } 348 return Digest; 349 } 350 351 std::string HexString; 352 }; 353 354 template <> struct MappingTraits<IncludeGraphNode> { 355 static void mapping(IO &IO, IncludeGraphNode &Node) { 356 IO.mapRequired("URI", Node.URI); 357 MappingNormalization<NormalizedSourceFlag, IncludeGraphNode::SourceFlag> 358 NSourceFlag(IO, Node.Flags); 359 IO.mapRequired("Flags", NSourceFlag->Flag); 360 MappingNormalization<NormalizedFileDigest, FileDigest> NDigest(IO, 361 Node.Digest); 362 IO.mapRequired("Digest", NDigest->HexString); 363 IO.mapRequired("DirectIncludes", Node.DirectIncludes); 364 } 365 }; 366 367 template <> struct MappingTraits<CompileCommandYAML> { 368 static void mapping(IO &IO, CompileCommandYAML &Cmd) { 369 IO.mapRequired("Directory", Cmd.Directory); 370 IO.mapRequired("CommandLine", Cmd.CommandLine); 371 } 372 }; 373 374 template <> struct MappingTraits<VariantEntry> { 375 static void mapping(IO &IO, VariantEntry &Variant) { 376 if (IO.mapTag("!Symbol", Variant.Symbol.hasValue())) { 377 if (!IO.outputting()) 378 Variant.Symbol.emplace(); 379 MappingTraits<Symbol>::mapping(IO, *Variant.Symbol); 380 } else if (IO.mapTag("!Refs", Variant.Refs.hasValue())) { 381 if (!IO.outputting()) 382 Variant.Refs.emplace(); 383 MappingTraits<RefBundle>::mapping(IO, *Variant.Refs); 384 } else if (IO.mapTag("!Relations", Variant.Relation.hasValue())) { 385 if (!IO.outputting()) 386 Variant.Relation.emplace(); 387 MappingTraits<Relation>::mapping(IO, *Variant.Relation); 388 } else if (IO.mapTag("!Source", Variant.Source.hasValue())) { 389 if (!IO.outputting()) 390 Variant.Source.emplace(); 391 MappingTraits<IncludeGraphNode>::mapping(IO, *Variant.Source); 392 } else if (IO.mapTag("!Cmd", Variant.Cmd.hasValue())) { 393 if (!IO.outputting()) 394 Variant.Cmd.emplace(); 395 MappingTraits<CompileCommandYAML>::mapping( 396 IO, static_cast<CompileCommandYAML &>(*Variant.Cmd)); 397 } 398 } 399 }; 400 401 } // namespace yaml 402 } // namespace llvm 403 404 namespace clang { 405 namespace clangd { 406 407 void writeYAML(const IndexFileOut &O, llvm::raw_ostream &OS) { 408 llvm::yaml::Output Yout(OS); 409 for (const auto &Sym : *O.Symbols) { 410 VariantEntry Entry; 411 Entry.Symbol = Sym; 412 Yout << Entry; 413 } 414 if (O.Refs) 415 for (auto &Sym : *O.Refs) { 416 VariantEntry Entry; 417 Entry.Refs = Sym; 418 Yout << Entry; 419 } 420 if (O.Relations) 421 for (auto &R : *O.Relations) { 422 VariantEntry Entry; 423 Entry.Relation = R; 424 Yout << Entry; 425 } 426 if (O.Sources) { 427 for (const auto &Source : *O.Sources) { 428 VariantEntry Entry; 429 Entry.Source = Source.getValue(); 430 Yout << Entry; 431 } 432 } 433 if (O.Cmd) { 434 VariantEntry Entry; 435 Entry.Cmd = *O.Cmd; 436 Yout << Entry; 437 } 438 } 439 440 llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data) { 441 SymbolSlab::Builder Symbols; 442 RefSlab::Builder Refs; 443 RelationSlab::Builder Relations; 444 llvm::BumpPtrAllocator 445 Arena; // store the underlying data of Position::FileURI. 446 llvm::UniqueStringSaver Strings(Arena); 447 llvm::yaml::Input Yin(Data, &Strings); 448 IncludeGraph Sources; 449 llvm::Optional<tooling::CompileCommand> Cmd; 450 while (Yin.setCurrentDocument()) { 451 llvm::yaml::EmptyContext Ctx; 452 VariantEntry Variant; 453 yamlize(Yin, Variant, true, Ctx); 454 if (Yin.error()) 455 return llvm::errorCodeToError(Yin.error()); 456 457 if (Variant.Symbol) 458 Symbols.insert(*Variant.Symbol); 459 if (Variant.Refs) 460 for (const auto &Ref : Variant.Refs->second) 461 Refs.insert(Variant.Refs->first, Ref); 462 if (Variant.Relation) 463 Relations.insert(*Variant.Relation); 464 if (Variant.Source) { 465 auto &IGN = Variant.Source.getValue(); 466 auto Entry = Sources.try_emplace(IGN.URI).first; 467 Entry->getValue() = std::move(IGN); 468 // Fixup refs to refer to map keys which will live on 469 Entry->getValue().URI = Entry->getKey(); 470 for (auto &Include : Entry->getValue().DirectIncludes) 471 Include = Sources.try_emplace(Include).first->getKey(); 472 } 473 if (Variant.Cmd) 474 Cmd = *Variant.Cmd; 475 Yin.nextDocument(); 476 } 477 478 IndexFileIn Result; 479 Result.Symbols.emplace(std::move(Symbols).build()); 480 Result.Refs.emplace(std::move(Refs).build()); 481 Result.Relations.emplace(std::move(Relations).build()); 482 if (Sources.size()) 483 Result.Sources = std::move(Sources); 484 Result.Cmd = std::move(Cmd); 485 return std::move(Result); 486 } 487 488 std::string toYAML(const Symbol &S) { 489 std::string Buf; 490 { 491 llvm::raw_string_ostream OS(Buf); 492 llvm::yaml::Output Yout(OS); 493 Symbol Sym = S; // copy: Yout<< requires mutability. 494 Yout << Sym; 495 } 496 return Buf; 497 } 498 499 std::string toYAML(const std::pair<SymbolID, llvm::ArrayRef<Ref>> &Data) { 500 RefBundle Refs = {Data.first, Data.second}; 501 std::string Buf; 502 { 503 llvm::raw_string_ostream OS(Buf); 504 llvm::yaml::Output Yout(OS); 505 Yout << Refs; 506 } 507 return Buf; 508 } 509 510 std::string toYAML(const Relation &R) { 511 std::string Buf; 512 { 513 llvm::raw_string_ostream OS(Buf); 514 llvm::yaml::Output Yout(OS); 515 Relation Rel = R; // copy: Yout<< requires mutability. 516 Yout << Rel; 517 } 518 return Buf; 519 } 520 521 std::string toYAML(const Ref &R) { 522 std::string Buf; 523 { 524 llvm::raw_string_ostream OS(Buf); 525 llvm::yaml::Output Yout(OS); 526 Ref Reference = R; // copy: Yout<< requires mutability. 527 Yout << Reference; 528 } 529 return Buf; 530 } 531 532 llvm::Expected<clangd::Symbol> 533 symbolFromYAML(StringRef YAML, llvm::UniqueStringSaver *Strings) { 534 clangd::Symbol Deserialized; 535 llvm::yaml::Input YAMLInput(YAML, Strings); 536 if (YAMLInput.error()) 537 return error("Unable to deserialize Symbol from YAML: {0}", YAML); 538 YAMLInput >> Deserialized; 539 return Deserialized; 540 } 541 542 llvm::Expected<clangd::Ref> refFromYAML(StringRef YAML, 543 llvm::UniqueStringSaver *Strings) { 544 clangd::Ref Deserialized; 545 llvm::yaml::Input YAMLInput(YAML, Strings); 546 if (YAMLInput.error()) 547 return error("Unable to deserialize Symbol from YAML: {0}", YAML); 548 YAMLInput >> Deserialized; 549 return Deserialized; 550 } 551 552 } // namespace clangd 553 } // namespace clang 554