1//===-- YAMLSerialization.cpp ------------------------------------*- C++-*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// A YAML index file is a sequence of tagged entries.
10// Each entry either encodes a Symbol or the list of references to a symbol
11// (a "ref bundle").
12//
13//===----------------------------------------------------------------------===//
14
15#include "Headers.h"
16#include "index/Ref.h"
17#include "index/Relation.h"
18#include "index/Serialization.h"
19#include "index/Symbol.h"
20#include "index/SymbolLocation.h"
21#include "index/SymbolOrigin.h"
22#include "clang/Tooling/CompilationDatabase.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/Support/Allocator.h"
25#include "llvm/Support/StringSaver.h"
26#include "llvm/Support/YAMLTraits.h"
27#include "llvm/Support/raw_ostream.h"
28#include <cstdint>
29#include <optional>
30
31namespace {
32struct YIncludeHeaderWithReferences;
33}
34
35LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences)
36LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Ref)
37LLVM_YAML_IS_SEQUENCE_VECTOR(YIncludeHeaderWithReferences)
38
39namespace {
40using RefBundle =
41 std::pair<clang::clangd::SymbolID, std::vector<clang::clangd::Ref>>;
42// This is a pale imitation of std::variant<Symbol, RefBundle, Relation>
43struct VariantEntry {
44 std::optional<clang::clangd::Symbol> Symbol;
45 std::optional<RefBundle> Refs;
46 std::optional<clang::clangd::Relation> Relation;
47 std::optional<clang::clangd::IncludeGraphNode> Source;
48 std::optional<clang::tooling::CompileCommand> Cmd;
49};
50// A class helps YAML to serialize the 32-bit encoded position (Line&Column),
51// as YAMLIO can't directly map bitfields.
52struct YPosition {
53 uint32_t Line;
54 uint32_t Column;
55};
56// A class helps YAML to serialize the IncludeHeaderWithReferences as YAMLIO
57// can't directly map bitfields.
58struct YIncludeHeaderWithReferences {
59 llvm::StringRef IncludeHeader;
60 uint32_t References;
61 clang::clangd::Symbol::IncludeDirective SupportedDirectives;
62
63 YIncludeHeaderWithReferences() = default;
64
65 YIncludeHeaderWithReferences(
66 llvm::StringRef IncludeHeader, uint32_t References,
67 clang::clangd::Symbol::IncludeDirective SupportedDirectives)
68 : IncludeHeader(IncludeHeader), References(References),
69 SupportedDirectives(SupportedDirectives) {}
70};
71
72// avoid ODR violation of specialization for non-owned CompileCommand
73struct CompileCommandYAML : clang::tooling::CompileCommand {};
74
75} // namespace
76namespace llvm {
77namespace yaml {
78
79using clang::clangd::FileDigest;
80using clang::clangd::IncludeGraph;
81using clang::clangd::IncludeGraphNode;
82using clang::clangd::Ref;
83using clang::clangd::RefKind;
84using clang::clangd::Relation;
85using clang::clangd::RelationKind;
86using clang::clangd::Symbol;
87using clang::clangd::SymbolID;
88using clang::clangd::SymbolLocation;
89using clang::index::SymbolInfo;
90using clang::index::SymbolKind;
91using clang::index::SymbolLanguage;
92using clang::tooling::CompileCommand;
93
94// Helper to (de)serialize the SymbolID. We serialize it as a hex string.
95struct NormalizedSymbolID {
96 NormalizedSymbolID(IO &) {}
97 NormalizedSymbolID(IO &, const SymbolID &ID) {
98 llvm::raw_string_ostream OS(HexString);
99 OS << ID;
100 }
101
102 SymbolID denormalize(IO &I) {
103 auto ID = SymbolID::fromStr(HexString);
104 if (!ID) {
105 I.setError(llvm::toString(E: ID.takeError()));
106 return SymbolID();
107 }
108 return *ID;
109 }
110
111 std::string HexString;
112};
113
114struct NormalizedSymbolFlag {
115 NormalizedSymbolFlag(IO &) {}
116 NormalizedSymbolFlag(IO &, Symbol::SymbolFlag F) {
117 Flag = static_cast<uint8_t>(F);
118 }
119
120 Symbol::SymbolFlag denormalize(IO &) {
121 return static_cast<Symbol::SymbolFlag>(Flag);
122 }
123
124 uint8_t Flag = 0;
125};
126
127template <> struct MappingTraits<YPosition> {
128 static void mapping(IO &IO, YPosition &Value) {
129 IO.mapRequired(Key: "Line", Val&: Value.Line);
130 IO.mapRequired(Key: "Column", Val&: Value.Column);
131 }
132};
133
134struct NormalizedPosition {
135 using Position = clang::clangd::SymbolLocation::Position;
136 NormalizedPosition(IO &) {}
137 NormalizedPosition(IO &, const Position &Pos) {
138 P.Line = Pos.line();
139 P.Column = Pos.column();
140 }
141
142 Position denormalize(IO &) {
143 Position Pos;
144 Pos.setLine(P.Line);
145 Pos.setColumn(P.Column);
146 return Pos;
147 }
148 YPosition P;
149};
150
151struct NormalizedFileURI {
152 NormalizedFileURI(IO &) {}
153 NormalizedFileURI(IO &, const char *FileURI) { URI = FileURI; }
154
155 const char *denormalize(IO &IO) {
156 assert(IO.getContext() &&
157 "Expecting an UniqueStringSaver to allocate data");
158 return static_cast<llvm::UniqueStringSaver *>(IO.getContext())
159 ->save(S: URI)
160 .data();
161 }
162
163 std::string URI;
164};
165
166template <> struct MappingTraits<SymbolLocation> {
167 static void mapping(IO &IO, SymbolLocation &Value) {
168 MappingNormalization<NormalizedFileURI, const char *> NFile(IO,
169 Value.FileURI);
170 IO.mapRequired(Key: "FileURI", Val&: NFile->URI);
171 MappingNormalization<NormalizedPosition, SymbolLocation::Position> NStart(
172 IO, Value.Start);
173 IO.mapRequired(Key: "Start", Val&: NStart->P);
174 MappingNormalization<NormalizedPosition, SymbolLocation::Position> NEnd(
175 IO, Value.End);
176 IO.mapRequired(Key: "End", Val&: NEnd->P);
177 }
178};
179
180template <> struct MappingTraits<SymbolInfo> {
181 static void mapping(IO &IO, SymbolInfo &SymInfo) {
182 // FIXME: expose other fields?
183 IO.mapRequired(Key: "Kind", Val&: SymInfo.Kind);
184 IO.mapRequired(Key: "Lang", Val&: SymInfo.Lang);
185 }
186};
187
188template <> struct ScalarBitSetTraits<clang::clangd::Symbol::IncludeDirective> {
189 static void bitset(IO &IO, clang::clangd::Symbol::IncludeDirective &Value) {
190 IO.bitSetCase(Val&: Value, Str: "Include", ConstVal: clang::clangd::Symbol::Include);
191 IO.bitSetCase(Val&: Value, Str: "Import", ConstVal: clang::clangd::Symbol::Import);
192 }
193};
194
195template <> struct MappingTraits<YIncludeHeaderWithReferences> {
196 static void mapping(IO &IO, YIncludeHeaderWithReferences &Inc) {
197 IO.mapRequired(Key: "Header", Val&: Inc.IncludeHeader);
198 IO.mapRequired(Key: "References", Val&: Inc.References);
199 IO.mapOptional(Key: "Directives", Val&: Inc.SupportedDirectives,
200 Default: clang::clangd::Symbol::Include);
201 }
202};
203
204struct NormalizedIncludeHeaders {
205 using IncludeHeader = clang::clangd::Symbol::IncludeHeaderWithReferences;
206 NormalizedIncludeHeaders(IO &) {}
207 NormalizedIncludeHeaders(
208 IO &, const llvm::SmallVector<IncludeHeader, 1> &IncludeHeaders) {
209 for (auto &I : IncludeHeaders) {
210 Headers.emplace_back(Args: I.IncludeHeader, Args: I.References,
211 Args: I.supportedDirectives());
212 }
213 }
214
215 llvm::SmallVector<IncludeHeader, 1> denormalize(IO &) {
216 llvm::SmallVector<IncludeHeader, 1> Result;
217 for (auto &H : Headers)
218 Result.emplace_back(Args&: H.IncludeHeader, Args&: H.References, Args&: H.SupportedDirectives);
219 return Result;
220 }
221 llvm::SmallVector<YIncludeHeaderWithReferences, 1> Headers;
222};
223
224template <> struct MappingTraits<Symbol> {
225 static void mapping(IO &IO, Symbol &Sym) {
226 MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, Sym.ID);
227 MappingNormalization<NormalizedSymbolFlag, Symbol::SymbolFlag> NSymbolFlag(
228 IO, Sym.Flags);
229 MappingNormalization<
230 NormalizedIncludeHeaders,
231 llvm::SmallVector<Symbol::IncludeHeaderWithReferences, 1>>
232 NIncludeHeaders(IO, Sym.IncludeHeaders);
233 IO.mapRequired(Key: "ID", Val&: NSymbolID->HexString);
234 IO.mapRequired(Key: "Name", Val&: Sym.Name);
235 IO.mapRequired(Key: "Scope", Val&: Sym.Scope);
236 IO.mapRequired(Key: "SymInfo", Val&: Sym.SymInfo);
237 IO.mapOptional(Key: "CanonicalDeclaration", Val&: Sym.CanonicalDeclaration,
238 Default: SymbolLocation());
239 IO.mapOptional(Key: "Definition", Val&: Sym.Definition, Default: SymbolLocation());
240 IO.mapOptional(Key: "References", Val&: Sym.References, Default: 0u);
241 IO.mapOptional(Key: "Flags", Val&: NSymbolFlag->Flag);
242 IO.mapOptional(Key: "Signature", Val&: Sym.Signature);
243 IO.mapOptional(Key: "TemplateSpecializationArgs",
244 Val&: Sym.TemplateSpecializationArgs);
245 IO.mapOptional(Key: "CompletionSnippetSuffix", Val&: Sym.CompletionSnippetSuffix);
246 IO.mapOptional(Key: "Documentation", Val&: Sym.Documentation);
247 IO.mapOptional(Key: "ReturnType", Val&: Sym.ReturnType);
248 IO.mapOptional(Key: "Type", Val&: Sym.Type);
249 IO.mapOptional(Key: "IncludeHeaders", Val&: NIncludeHeaders->Headers);
250 }
251};
252
253template <> struct ScalarEnumerationTraits<SymbolLanguage> {
254 static void enumeration(IO &IO, SymbolLanguage &Value) {
255 IO.enumCase(Val&: Value, Str: "C", ConstVal: SymbolLanguage::C);
256 IO.enumCase(Val&: Value, Str: "Cpp", ConstVal: SymbolLanguage::CXX);
257 IO.enumCase(Val&: Value, Str: "ObjC", ConstVal: SymbolLanguage::ObjC);
258 IO.enumCase(Val&: Value, Str: "Swift", ConstVal: SymbolLanguage::Swift);
259 }
260};
261
262template <> struct ScalarEnumerationTraits<SymbolKind> {
263 static void enumeration(IO &IO, SymbolKind &Value) {
264#define DEFINE_ENUM(name) IO.enumCase(Value, #name, SymbolKind::name)
265
266 DEFINE_ENUM(Unknown);
267 DEFINE_ENUM(Function);
268 DEFINE_ENUM(Module);
269 DEFINE_ENUM(Namespace);
270 DEFINE_ENUM(NamespaceAlias);
271 DEFINE_ENUM(Macro);
272 DEFINE_ENUM(Enum);
273 DEFINE_ENUM(Struct);
274 DEFINE_ENUM(Class);
275 DEFINE_ENUM(Protocol);
276 DEFINE_ENUM(Extension);
277 DEFINE_ENUM(Union);
278 DEFINE_ENUM(TypeAlias);
279 DEFINE_ENUM(Function);
280 DEFINE_ENUM(Variable);
281 DEFINE_ENUM(Field);
282 DEFINE_ENUM(EnumConstant);
283 DEFINE_ENUM(InstanceMethod);
284 DEFINE_ENUM(ClassMethod);
285 DEFINE_ENUM(StaticMethod);
286 DEFINE_ENUM(InstanceProperty);
287 DEFINE_ENUM(ClassProperty);
288 DEFINE_ENUM(StaticProperty);
289 DEFINE_ENUM(Constructor);
290 DEFINE_ENUM(Destructor);
291 DEFINE_ENUM(ConversionFunction);
292 DEFINE_ENUM(Parameter);
293 DEFINE_ENUM(Using);
294
295#undef DEFINE_ENUM
296 }
297};
298
299template <> struct MappingTraits<RefBundle> {
300 static void mapping(IO &IO, RefBundle &Refs) {
301 MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO,
302 Refs.first);
303 IO.mapRequired(Key: "ID", Val&: NSymbolID->HexString);
304 IO.mapRequired(Key: "References", Val&: Refs.second);
305 }
306};
307
308struct NormalizedRefKind {
309 NormalizedRefKind(IO &) {}
310 NormalizedRefKind(IO &, RefKind O) { Kind = static_cast<uint8_t>(O); }
311
312 RefKind denormalize(IO &) { return static_cast<RefKind>(Kind); }
313
314 uint8_t Kind = 0;
315};
316
317template <> struct MappingTraits<Ref> {
318 static void mapping(IO &IO, Ref &R) {
319 MappingNormalization<NormalizedRefKind, RefKind> NKind(IO, R.Kind);
320 IO.mapRequired(Key: "Kind", Val&: NKind->Kind);
321 IO.mapRequired(Key: "Location", Val&: R.Location);
322 IO.mapOptional(Key: "Container", Val&: R.Container);
323 }
324};
325
326struct NormalizedSymbolRole {
327 NormalizedSymbolRole(IO &) {}
328 NormalizedSymbolRole(IO &IO, RelationKind R) {
329 Kind = static_cast<uint8_t>(R);
330 }
331
332 RelationKind denormalize(IO &IO) { return static_cast<RelationKind>(Kind); }
333
334 uint8_t Kind = 0;
335};
336
337template <> struct MappingTraits<SymbolID> {
338 static void mapping(IO &IO, SymbolID &ID) {
339 MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, ID);
340 IO.mapRequired(Key: "ID", Val&: NSymbolID->HexString);
341 }
342};
343
344template <> struct MappingTraits<Relation> {
345 static void mapping(IO &IO, Relation &Relation) {
346 MappingNormalization<NormalizedSymbolRole, RelationKind> NRole(
347 IO, Relation.Predicate);
348 IO.mapRequired(Key: "Subject", Val&: Relation.Subject);
349 IO.mapRequired(Key: "Predicate", Val&: NRole->Kind);
350 IO.mapRequired(Key: "Object", Val&: Relation.Object);
351 }
352};
353
354struct NormalizedSourceFlag {
355 NormalizedSourceFlag(IO &) {}
356 NormalizedSourceFlag(IO &, IncludeGraphNode::SourceFlag O) {
357 Flag = static_cast<uint8_t>(O);
358 }
359
360 IncludeGraphNode::SourceFlag denormalize(IO &) {
361 return static_cast<IncludeGraphNode::SourceFlag>(Flag);
362 }
363
364 uint8_t Flag = 0;
365};
366
367struct NormalizedFileDigest {
368 NormalizedFileDigest(IO &) {}
369 NormalizedFileDigest(IO &, const FileDigest &Digest) {
370 HexString = llvm::toHex(Input: Digest);
371 }
372
373 FileDigest denormalize(IO &I) {
374 FileDigest Digest;
375 if (HexString.size() == Digest.size() * 2 &&
376 llvm::all_of(Range&: HexString, P: llvm::isHexDigit)) {
377 memcpy(dest: Digest.data(), src: llvm::fromHex(Input: HexString).data(), n: Digest.size());
378 } else {
379 I.setError(std::string("Bad hex file digest: ") + HexString);
380 }
381 return Digest;
382 }
383
384 std::string HexString;
385};
386
387template <> struct MappingTraits<IncludeGraphNode> {
388 static void mapping(IO &IO, IncludeGraphNode &Node) {
389 IO.mapRequired(Key: "URI", Val&: Node.URI);
390 MappingNormalization<NormalizedSourceFlag, IncludeGraphNode::SourceFlag>
391 NSourceFlag(IO, Node.Flags);
392 IO.mapRequired(Key: "Flags", Val&: NSourceFlag->Flag);
393 MappingNormalization<NormalizedFileDigest, FileDigest> NDigest(IO,
394 Node.Digest);
395 IO.mapRequired(Key: "Digest", Val&: NDigest->HexString);
396 IO.mapRequired(Key: "DirectIncludes", Val&: Node.DirectIncludes);
397 }
398};
399
400template <> struct MappingTraits<CompileCommandYAML> {
401 static void mapping(IO &IO, CompileCommandYAML &Cmd) {
402 IO.mapRequired(Key: "Directory", Val&: Cmd.Directory);
403 IO.mapRequired(Key: "CommandLine", Val&: Cmd.CommandLine);
404 }
405};
406
407template <> struct MappingTraits<VariantEntry> {
408 static void mapping(IO &IO, VariantEntry &Variant) {
409 if (IO.mapTag(Tag: "!Symbol", Default: Variant.Symbol.has_value())) {
410 if (!IO.outputting())
411 Variant.Symbol.emplace();
412 MappingTraits<Symbol>::mapping(IO, Sym&: *Variant.Symbol);
413 } else if (IO.mapTag(Tag: "!Refs", Default: Variant.Refs.has_value())) {
414 if (!IO.outputting())
415 Variant.Refs.emplace();
416 MappingTraits<RefBundle>::mapping(IO, Refs&: *Variant.Refs);
417 } else if (IO.mapTag(Tag: "!Relations", Default: Variant.Relation.has_value())) {
418 if (!IO.outputting())
419 Variant.Relation.emplace();
420 MappingTraits<Relation>::mapping(IO, Relation&: *Variant.Relation);
421 } else if (IO.mapTag(Tag: "!Source", Default: Variant.Source.has_value())) {
422 if (!IO.outputting())
423 Variant.Source.emplace();
424 MappingTraits<IncludeGraphNode>::mapping(IO, Node&: *Variant.Source);
425 } else if (IO.mapTag(Tag: "!Cmd", Default: Variant.Cmd.has_value())) {
426 if (!IO.outputting())
427 Variant.Cmd.emplace();
428 MappingTraits<CompileCommandYAML>::mapping(
429 IO, Cmd&: static_cast<CompileCommandYAML &>(*Variant.Cmd));
430 }
431 }
432};
433
434} // namespace yaml
435} // namespace llvm
436
437namespace clang {
438namespace clangd {
439
440void writeYAML(const IndexFileOut &O, llvm::raw_ostream &OS) {
441 llvm::yaml::Output Yout(OS);
442 for (const auto &Sym : *O.Symbols) {
443 VariantEntry Entry;
444 Entry.Symbol = Sym;
445 Yout << Entry;
446 }
447 if (O.Refs)
448 for (auto &Sym : *O.Refs) {
449 VariantEntry Entry;
450 Entry.Refs = Sym;
451 Yout << Entry;
452 }
453 if (O.Relations)
454 for (auto &R : *O.Relations) {
455 VariantEntry Entry;
456 Entry.Relation = R;
457 Yout << Entry;
458 }
459 if (O.Sources) {
460 for (const auto &Source : *O.Sources) {
461 VariantEntry Entry;
462 Entry.Source = Source.getValue();
463 Yout << Entry;
464 }
465 }
466 if (O.Cmd) {
467 VariantEntry Entry;
468 Entry.Cmd = *O.Cmd;
469 Yout << Entry;
470 }
471}
472
473llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data,
474 SymbolOrigin Origin) {
475 SymbolSlab::Builder Symbols;
476 RefSlab::Builder Refs;
477 RelationSlab::Builder Relations;
478 llvm::BumpPtrAllocator
479 Arena; // store the underlying data of Position::FileURI.
480 llvm::UniqueStringSaver Strings(Arena);
481 llvm::yaml::Input Yin(Data, &Strings);
482 IncludeGraph Sources;
483 std::optional<tooling::CompileCommand> Cmd;
484 while (Yin.setCurrentDocument()) {
485 llvm::yaml::EmptyContext Ctx;
486 VariantEntry Variant;
487 yamlize(io&: Yin, Val&: Variant, true, Ctx);
488 if (Yin.error())
489 return llvm::errorCodeToError(EC: Yin.error());
490
491 if (Variant.Symbol) {
492 Variant.Symbol->Origin = Origin;
493 Symbols.insert(S: *Variant.Symbol);
494 }
495 if (Variant.Refs)
496 for (const auto &Ref : Variant.Refs->second)
497 Refs.insert(ID: Variant.Refs->first, S: Ref);
498 if (Variant.Relation)
499 Relations.insert(R: *Variant.Relation);
500 if (Variant.Source) {
501 auto &IGN = *Variant.Source;
502 auto Entry = Sources.try_emplace(Key: IGN.URI).first;
503 Entry->getValue() = std::move(IGN);
504 // Fixup refs to refer to map keys which will live on
505 Entry->getValue().URI = Entry->getKey();
506 for (auto &Include : Entry->getValue().DirectIncludes)
507 Include = Sources.try_emplace(Key: Include).first->getKey();
508 }
509 if (Variant.Cmd)
510 Cmd = *Variant.Cmd;
511 Yin.nextDocument();
512 }
513
514 IndexFileIn Result;
515 Result.Symbols.emplace(args: std::move(Symbols).build());
516 Result.Refs.emplace(args: std::move(Refs).build());
517 Result.Relations.emplace(args: std::move(Relations).build());
518 if (Sources.size())
519 Result.Sources = std::move(Sources);
520 Result.Cmd = std::move(Cmd);
521 return std::move(Result);
522}
523
524std::string toYAML(const Symbol &S) {
525 std::string Buf;
526 {
527 llvm::raw_string_ostream OS(Buf);
528 llvm::yaml::Output Yout(OS);
529 Symbol Sym = S; // copy: Yout<< requires mutability.
530 Yout << Sym;
531 }
532 return Buf;
533}
534
535std::string toYAML(const std::pair<SymbolID, llvm::ArrayRef<Ref>> &Data) {
536 RefBundle Refs = {Data.first, Data.second};
537 std::string Buf;
538 {
539 llvm::raw_string_ostream OS(Buf);
540 llvm::yaml::Output Yout(OS);
541 Yout << Refs;
542 }
543 return Buf;
544}
545
546std::string toYAML(const Relation &R) {
547 std::string Buf;
548 {
549 llvm::raw_string_ostream OS(Buf);
550 llvm::yaml::Output Yout(OS);
551 Relation Rel = R; // copy: Yout<< requires mutability.
552 Yout << Rel;
553 }
554 return Buf;
555}
556
557std::string toYAML(const Ref &R) {
558 std::string Buf;
559 {
560 llvm::raw_string_ostream OS(Buf);
561 llvm::yaml::Output Yout(OS);
562 Ref Reference = R; // copy: Yout<< requires mutability.
563 Yout << Reference;
564 }
565 return Buf;
566}
567
568} // namespace clangd
569} // namespace clang
570

source code of clang-tools-extra/clangd/index/YAMLSerialization.cpp