1//===-- YAMLSerialization.cpp ------------------------------------*- C++-*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// A YAML index file is a sequence of tagged entries.
10// Each entry either encodes a Symbol or the list of references to a symbol
11// (a "ref bundle").
12//
13//===----------------------------------------------------------------------===//
14
15#include "Headers.h"
16#include "index/Ref.h"
17#include "index/Relation.h"
18#include "index/Serialization.h"
19#include "index/Symbol.h"
20#include "index/SymbolLocation.h"
21#include "index/SymbolOrigin.h"
22#include "clang/Tooling/CompilationDatabase.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/Support/Allocator.h"
25#include "llvm/Support/StringSaver.h"
26#include "llvm/Support/YAMLTraits.h"
27#include "llvm/Support/raw_ostream.h"
28#include <cstdint>
29#include <optional>
30
31namespace {
32struct YIncludeHeaderWithReferences;
33}
34
35LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences)
36LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Ref)
37LLVM_YAML_IS_SEQUENCE_VECTOR(YIncludeHeaderWithReferences)
38
39namespace {
40using RefBundle =
41 std::pair<clang::clangd::SymbolID, std::vector<clang::clangd::Ref>>;
42// This is a pale imitation of std::variant<Symbol, RefBundle, Relation>
43struct VariantEntry {
44 std::optional<clang::clangd::Symbol> Symbol;
45 std::optional<RefBundle> Refs;
46 std::optional<clang::clangd::Relation> Relation;
47 std::optional<clang::clangd::IncludeGraphNode> Source;
48 std::optional<clang::tooling::CompileCommand> Cmd;
49};
50// A class helps YAML to serialize the 32-bit encoded position (Line&Column),
51// as YAMLIO can't directly map bitfields.
52struct YPosition {
53 uint32_t Line;
54 uint32_t Column;
55};
56// A class helps YAML to serialize the IncludeHeaderWithReferences as YAMLIO
57// can't directly map bitfields.
58struct YIncludeHeaderWithReferences {
59 llvm::StringRef IncludeHeader;
60 uint32_t References;
61 clang::clangd::Symbol::IncludeDirective SupportedDirectives;
62
63 YIncludeHeaderWithReferences() = default;
64
65 YIncludeHeaderWithReferences(
66 llvm::StringRef IncludeHeader, uint32_t References,
67 clang::clangd::Symbol::IncludeDirective SupportedDirectives)
68 : IncludeHeader(IncludeHeader), References(References),
69 SupportedDirectives(SupportedDirectives) {}
70};
71
72// avoid ODR violation of specialization for non-owned CompileCommand
73struct CompileCommandYAML : clang::tooling::CompileCommand {};
74
75} // namespace
76namespace llvm {
77namespace yaml {
78
79using clang::clangd::FileDigest;
80using clang::clangd::IncludeGraph;
81using clang::clangd::IncludeGraphNode;
82using clang::clangd::Ref;
83using clang::clangd::RefKind;
84using clang::clangd::Relation;
85using clang::clangd::RelationKind;
86using clang::clangd::Symbol;
87using clang::clangd::SymbolID;
88using clang::clangd::SymbolLocation;
89using clang::index::SymbolInfo;
90using clang::index::SymbolKind;
91using clang::index::SymbolLanguage;
92using clang::tooling::CompileCommand;
93
94// Helper to (de)serialize the SymbolID. We serialize it as a hex string.
95struct NormalizedSymbolID {
96 NormalizedSymbolID(IO &) {}
97 NormalizedSymbolID(IO &, const SymbolID &ID) {
98 llvm::raw_string_ostream OS(HexString);
99 OS << ID;
100 }
101
102 SymbolID denormalize(IO &I) {
103 auto ID = SymbolID::fromStr(HexString);
104 if (!ID) {
105 I.setError(llvm::toString(E: ID.takeError()));
106 return SymbolID();
107 }
108 return *ID;
109 }
110
111 std::string HexString;
112};
113
114struct NormalizedSymbolFlag {
115 NormalizedSymbolFlag(IO &) {}
116 NormalizedSymbolFlag(IO &, Symbol::SymbolFlag F) {
117 Flag = static_cast<uint8_t>(F);
118 }
119
120 Symbol::SymbolFlag denormalize(IO &) {
121 return static_cast<Symbol::SymbolFlag>(Flag);
122 }
123
124 uint8_t Flag = 0;
125};
126
127template <> struct MappingTraits<YPosition> {
128 static void mapping(IO &IO, YPosition &Value) {
129 IO.mapRequired(Key: "Line", Val&: Value.Line);
130 IO.mapRequired(Key: "Column", Val&: Value.Column);
131 }
132};
133
134struct NormalizedPosition {
135 using Position = clang::clangd::SymbolLocation::Position;
136 NormalizedPosition(IO &) {}
137 NormalizedPosition(IO &, const Position &Pos) {
138 P.Line = Pos.line();
139 P.Column = Pos.column();
140 }
141
142 Position denormalize(IO &) {
143 Position Pos;
144 Pos.setLine(P.Line);
145 Pos.setColumn(P.Column);
146 return Pos;
147 }
148 YPosition P;
149};
150
151struct NormalizedFileURI {
152 NormalizedFileURI(IO &) {}
153 NormalizedFileURI(IO &, const char *FileURI) { URI = FileURI; }
154
155 const char *denormalize(IO &IO) {
156 assert(IO.getContext() &&
157 "Expecting an UniqueStringSaver to allocate data");
158 return static_cast<llvm::UniqueStringSaver *>(IO.getContext())
159 ->save(S: URI)
160 .data();
161 }
162
163 std::string URI;
164};
165
166template <> struct MappingTraits<SymbolLocation> {
167 static void mapping(IO &IO, SymbolLocation &Value) {
168 MappingNormalization<NormalizedFileURI, const char *> NFile(IO,
169 Value.FileURI);
170 IO.mapRequired(Key: "FileURI", Val&: NFile->URI);
171 MappingNormalization<NormalizedPosition, SymbolLocation::Position> NStart(
172 IO, Value.Start);
173 IO.mapRequired(Key: "Start", Val&: NStart->P);
174 MappingNormalization<NormalizedPosition, SymbolLocation::Position> NEnd(
175 IO, Value.End);
176 IO.mapRequired(Key: "End", Val&: NEnd->P);
177 }
178};
179
180template <> struct MappingTraits<SymbolInfo> {
181 static void mapping(IO &IO, SymbolInfo &SymInfo) {
182 // FIXME: expose other fields?
183 IO.mapRequired(Key: "Kind", Val&: SymInfo.Kind);
184 IO.mapRequired(Key: "Lang", Val&: SymInfo.Lang);
185 }
186};
187
188template <> struct ScalarBitSetTraits<clang::clangd::Symbol::IncludeDirective> {
189 static void bitset(IO &IO, clang::clangd::Symbol::IncludeDirective &Value) {
190 IO.bitSetCase(Val&: Value, Str: "Include", ConstVal: clang::clangd::Symbol::Include);
191 IO.bitSetCase(Val&: Value, Str: "Import", ConstVal: clang::clangd::Symbol::Import);
192 }
193};
194
195template <> struct MappingTraits<YIncludeHeaderWithReferences> {
196 static void mapping(IO &IO, YIncludeHeaderWithReferences &Inc) {
197 IO.mapRequired(Key: "Header", Val&: Inc.IncludeHeader);
198 IO.mapRequired(Key: "References", Val&: Inc.References);
199 IO.mapOptional(Key: "Directives", Val&: Inc.SupportedDirectives,
200 Default: clang::clangd::Symbol::Include);
201 }
202};
203
204struct NormalizedIncludeHeaders {
205 using IncludeHeader = clang::clangd::Symbol::IncludeHeaderWithReferences;
206 NormalizedIncludeHeaders(IO &) {}
207 NormalizedIncludeHeaders(
208 IO &, const llvm::SmallVector<IncludeHeader, 1> &IncludeHeaders) {
209 for (auto &I : IncludeHeaders) {
210 Headers.emplace_back(Args: I.IncludeHeader, Args: I.References,
211 Args: I.supportedDirectives());
212 }
213 }
214
215 llvm::SmallVector<IncludeHeader, 1> denormalize(IO &) {
216 llvm::SmallVector<IncludeHeader, 1> Result;
217 for (auto &H : Headers)
218 Result.emplace_back(Args&: H.IncludeHeader, Args&: H.References, Args&: H.SupportedDirectives);
219 return Result;
220 }
221 llvm::SmallVector<YIncludeHeaderWithReferences, 1> Headers;
222};
223
224template <> struct MappingTraits<Symbol> {
225 static void mapping(IO &IO, Symbol &Sym) {
226 MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, Sym.ID);
227 MappingNormalization<NormalizedSymbolFlag, Symbol::SymbolFlag> NSymbolFlag(
228 IO, Sym.Flags);
229 MappingNormalization<
230 NormalizedIncludeHeaders,
231 llvm::SmallVector<Symbol::IncludeHeaderWithReferences, 1>>
232 NIncludeHeaders(IO, Sym.IncludeHeaders);
233 IO.mapRequired(Key: "ID", Val&: NSymbolID->HexString);
234 IO.mapRequired(Key: "Name", Val&: Sym.Name);
235 IO.mapRequired(Key: "Scope", Val&: Sym.Scope);
236 IO.mapRequired(Key: "SymInfo", Val&: Sym.SymInfo);
237 IO.mapOptional(Key: "CanonicalDeclaration", Val&: Sym.CanonicalDeclaration,
238 Default: SymbolLocation());
239 IO.mapOptional(Key: "Definition", Val&: Sym.Definition, Default: SymbolLocation());
240 IO.mapOptional(Key: "References", Val&: Sym.References, Default: 0u);
241 IO.mapOptional(Key: "Flags", Val&: NSymbolFlag->Flag);
242 IO.mapOptional(Key: "Signature", Val&: Sym.Signature);
243 IO.mapOptional(Key: "TemplateSpecializationArgs",
244 Val&: Sym.TemplateSpecializationArgs);
245 IO.mapOptional(Key: "CompletionSnippetSuffix", Val&: Sym.CompletionSnippetSuffix);
246 IO.mapOptional(Key: "Documentation", Val&: Sym.Documentation);
247 IO.mapOptional(Key: "ReturnType", Val&: Sym.ReturnType);
248 IO.mapOptional(Key: "Type", Val&: Sym.Type);
249 IO.mapOptional(Key: "IncludeHeaders", Val&: NIncludeHeaders->Headers);
250 }
251};
252
253template <> struct ScalarEnumerationTraits<SymbolLanguage> {
254 static void enumeration(IO &IO, SymbolLanguage &Value) {
255 IO.enumCase(Val&: Value, Str: "C", ConstVal: SymbolLanguage::C);
256 IO.enumCase(Val&: Value, Str: "Cpp", ConstVal: SymbolLanguage::CXX);
257 IO.enumCase(Val&: Value, Str: "ObjC", ConstVal: SymbolLanguage::ObjC);
258 IO.enumCase(Val&: Value, Str: "Swift", ConstVal: SymbolLanguage::Swift);
259 }
260};
261
262template <> struct ScalarEnumerationTraits<SymbolKind> {
263 static void enumeration(IO &IO, SymbolKind &Value) {
264#define DEFINE_ENUM(name) IO.enumCase(Value, #name, SymbolKind::name)
265
266 DEFINE_ENUM(Unknown);
267 DEFINE_ENUM(Function);
268 DEFINE_ENUM(Module);
269 DEFINE_ENUM(Namespace);
270 DEFINE_ENUM(NamespaceAlias);
271 DEFINE_ENUM(Macro);
272 DEFINE_ENUM(Enum);
273 DEFINE_ENUM(Struct);
274 DEFINE_ENUM(Class);
275 DEFINE_ENUM(Protocol);
276 DEFINE_ENUM(Extension);
277 DEFINE_ENUM(Union);
278 DEFINE_ENUM(TypeAlias);
279 DEFINE_ENUM(Function);
280 DEFINE_ENUM(Variable);
281 DEFINE_ENUM(Field);
282 DEFINE_ENUM(EnumConstant);
283 DEFINE_ENUM(InstanceMethod);
284 DEFINE_ENUM(ClassMethod);
285 DEFINE_ENUM(StaticMethod);
286 DEFINE_ENUM(InstanceProperty);
287 DEFINE_ENUM(ClassProperty);
288 DEFINE_ENUM(StaticProperty);
289 DEFINE_ENUM(Constructor);
290 DEFINE_ENUM(Destructor);
291 DEFINE_ENUM(ConversionFunction);
292 DEFINE_ENUM(Parameter);
293 DEFINE_ENUM(Using);
294
295#undef DEFINE_ENUM
296 }
297};
298
299template <> struct MappingTraits<RefBundle> {
300 static void mapping(IO &IO, RefBundle &Refs) {
301 MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO,
302 Refs.first);
303 IO.mapRequired(Key: "ID", Val&: NSymbolID->HexString);
304 IO.mapRequired(Key: "References", Val&: Refs.second);
305 }
306};
307
308struct NormalizedRefKind {
309 NormalizedRefKind(IO &) {}
310 NormalizedRefKind(IO &, RefKind O) { Kind = static_cast<uint8_t>(O); }
311
312 RefKind denormalize(IO &) { return static_cast<RefKind>(Kind); }
313
314 uint8_t Kind = 0;
315};
316
317template <> struct MappingTraits<Ref> {
318 static void mapping(IO &IO, Ref &R) {
319 MappingNormalization<NormalizedRefKind, RefKind> NKind(IO, R.Kind);
320 IO.mapRequired(Key: "Kind", Val&: NKind->Kind);
321 IO.mapRequired(Key: "Location", Val&: R.Location);
322 }
323};
324
325struct NormalizedSymbolRole {
326 NormalizedSymbolRole(IO &) {}
327 NormalizedSymbolRole(IO &IO, RelationKind R) {
328 Kind = static_cast<uint8_t>(R);
329 }
330
331 RelationKind denormalize(IO &IO) { return static_cast<RelationKind>(Kind); }
332
333 uint8_t Kind = 0;
334};
335
336template <> struct MappingTraits<SymbolID> {
337 static void mapping(IO &IO, SymbolID &ID) {
338 MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, ID);
339 IO.mapRequired(Key: "ID", Val&: NSymbolID->HexString);
340 }
341};
342
343template <> struct MappingTraits<Relation> {
344 static void mapping(IO &IO, Relation &Relation) {
345 MappingNormalization<NormalizedSymbolRole, RelationKind> NRole(
346 IO, Relation.Predicate);
347 IO.mapRequired(Key: "Subject", Val&: Relation.Subject);
348 IO.mapRequired(Key: "Predicate", Val&: NRole->Kind);
349 IO.mapRequired(Key: "Object", Val&: Relation.Object);
350 }
351};
352
353struct NormalizedSourceFlag {
354 NormalizedSourceFlag(IO &) {}
355 NormalizedSourceFlag(IO &, IncludeGraphNode::SourceFlag O) {
356 Flag = static_cast<uint8_t>(O);
357 }
358
359 IncludeGraphNode::SourceFlag denormalize(IO &) {
360 return static_cast<IncludeGraphNode::SourceFlag>(Flag);
361 }
362
363 uint8_t Flag = 0;
364};
365
366struct NormalizedFileDigest {
367 NormalizedFileDigest(IO &) {}
368 NormalizedFileDigest(IO &, const FileDigest &Digest) {
369 HexString = llvm::toHex(Input: Digest);
370 }
371
372 FileDigest denormalize(IO &I) {
373 FileDigest Digest;
374 if (HexString.size() == Digest.size() * 2 &&
375 llvm::all_of(Range&: HexString, P: llvm::isHexDigit)) {
376 memcpy(dest: Digest.data(), src: llvm::fromHex(Input: HexString).data(), n: Digest.size());
377 } else {
378 I.setError(std::string("Bad hex file digest: ") + HexString);
379 }
380 return Digest;
381 }
382
383 std::string HexString;
384};
385
386template <> struct MappingTraits<IncludeGraphNode> {
387 static void mapping(IO &IO, IncludeGraphNode &Node) {
388 IO.mapRequired(Key: "URI", Val&: Node.URI);
389 MappingNormalization<NormalizedSourceFlag, IncludeGraphNode::SourceFlag>
390 NSourceFlag(IO, Node.Flags);
391 IO.mapRequired(Key: "Flags", Val&: NSourceFlag->Flag);
392 MappingNormalization<NormalizedFileDigest, FileDigest> NDigest(IO,
393 Node.Digest);
394 IO.mapRequired(Key: "Digest", Val&: NDigest->HexString);
395 IO.mapRequired(Key: "DirectIncludes", Val&: Node.DirectIncludes);
396 }
397};
398
399template <> struct MappingTraits<CompileCommandYAML> {
400 static void mapping(IO &IO, CompileCommandYAML &Cmd) {
401 IO.mapRequired(Key: "Directory", Val&: Cmd.Directory);
402 IO.mapRequired(Key: "CommandLine", Val&: Cmd.CommandLine);
403 }
404};
405
406template <> struct MappingTraits<VariantEntry> {
407 static void mapping(IO &IO, VariantEntry &Variant) {
408 if (IO.mapTag(Tag: "!Symbol", Default: Variant.Symbol.has_value())) {
409 if (!IO.outputting())
410 Variant.Symbol.emplace();
411 MappingTraits<Symbol>::mapping(IO, Sym&: *Variant.Symbol);
412 } else if (IO.mapTag(Tag: "!Refs", Default: Variant.Refs.has_value())) {
413 if (!IO.outputting())
414 Variant.Refs.emplace();
415 MappingTraits<RefBundle>::mapping(IO, Refs&: *Variant.Refs);
416 } else if (IO.mapTag(Tag: "!Relations", Default: Variant.Relation.has_value())) {
417 if (!IO.outputting())
418 Variant.Relation.emplace();
419 MappingTraits<Relation>::mapping(IO, Relation&: *Variant.Relation);
420 } else if (IO.mapTag(Tag: "!Source", Default: Variant.Source.has_value())) {
421 if (!IO.outputting())
422 Variant.Source.emplace();
423 MappingTraits<IncludeGraphNode>::mapping(IO, Node&: *Variant.Source);
424 } else if (IO.mapTag(Tag: "!Cmd", Default: Variant.Cmd.has_value())) {
425 if (!IO.outputting())
426 Variant.Cmd.emplace();
427 MappingTraits<CompileCommandYAML>::mapping(
428 IO, Cmd&: static_cast<CompileCommandYAML &>(*Variant.Cmd));
429 }
430 }
431};
432
433} // namespace yaml
434} // namespace llvm
435
436namespace clang {
437namespace clangd {
438
439void writeYAML(const IndexFileOut &O, llvm::raw_ostream &OS) {
440 llvm::yaml::Output Yout(OS);
441 for (const auto &Sym : *O.Symbols) {
442 VariantEntry Entry;
443 Entry.Symbol = Sym;
444 Yout << Entry;
445 }
446 if (O.Refs)
447 for (auto &Sym : *O.Refs) {
448 VariantEntry Entry;
449 Entry.Refs = Sym;
450 Yout << Entry;
451 }
452 if (O.Relations)
453 for (auto &R : *O.Relations) {
454 VariantEntry Entry;
455 Entry.Relation = R;
456 Yout << Entry;
457 }
458 if (O.Sources) {
459 for (const auto &Source : *O.Sources) {
460 VariantEntry Entry;
461 Entry.Source = Source.getValue();
462 Yout << Entry;
463 }
464 }
465 if (O.Cmd) {
466 VariantEntry Entry;
467 Entry.Cmd = *O.Cmd;
468 Yout << Entry;
469 }
470}
471
472llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data,
473 SymbolOrigin Origin) {
474 SymbolSlab::Builder Symbols;
475 RefSlab::Builder Refs;
476 RelationSlab::Builder Relations;
477 llvm::BumpPtrAllocator
478 Arena; // store the underlying data of Position::FileURI.
479 llvm::UniqueStringSaver Strings(Arena);
480 llvm::yaml::Input Yin(Data, &Strings);
481 IncludeGraph Sources;
482 std::optional<tooling::CompileCommand> Cmd;
483 while (Yin.setCurrentDocument()) {
484 llvm::yaml::EmptyContext Ctx;
485 VariantEntry Variant;
486 yamlize(io&: Yin, Val&: Variant, true, Ctx);
487 if (Yin.error())
488 return llvm::errorCodeToError(EC: Yin.error());
489
490 if (Variant.Symbol) {
491 Variant.Symbol->Origin = Origin;
492 Symbols.insert(S: *Variant.Symbol);
493 }
494 if (Variant.Refs)
495 for (const auto &Ref : Variant.Refs->second)
496 Refs.insert(ID: Variant.Refs->first, S: Ref);
497 if (Variant.Relation)
498 Relations.insert(R: *Variant.Relation);
499 if (Variant.Source) {
500 auto &IGN = *Variant.Source;
501 auto Entry = Sources.try_emplace(Key: IGN.URI).first;
502 Entry->getValue() = std::move(IGN);
503 // Fixup refs to refer to map keys which will live on
504 Entry->getValue().URI = Entry->getKey();
505 for (auto &Include : Entry->getValue().DirectIncludes)
506 Include = Sources.try_emplace(Key: Include).first->getKey();
507 }
508 if (Variant.Cmd)
509 Cmd = *Variant.Cmd;
510 Yin.nextDocument();
511 }
512
513 IndexFileIn Result;
514 Result.Symbols.emplace(args: std::move(Symbols).build());
515 Result.Refs.emplace(args: std::move(Refs).build());
516 Result.Relations.emplace(args: std::move(Relations).build());
517 if (Sources.size())
518 Result.Sources = std::move(Sources);
519 Result.Cmd = std::move(Cmd);
520 return std::move(Result);
521}
522
523std::string toYAML(const Symbol &S) {
524 std::string Buf;
525 {
526 llvm::raw_string_ostream OS(Buf);
527 llvm::yaml::Output Yout(OS);
528 Symbol Sym = S; // copy: Yout<< requires mutability.
529 Yout << Sym;
530 }
531 return Buf;
532}
533
534std::string toYAML(const std::pair<SymbolID, llvm::ArrayRef<Ref>> &Data) {
535 RefBundle Refs = {Data.first, Data.second};
536 std::string Buf;
537 {
538 llvm::raw_string_ostream OS(Buf);
539 llvm::yaml::Output Yout(OS);
540 Yout << Refs;
541 }
542 return Buf;
543}
544
545std::string toYAML(const Relation &R) {
546 std::string Buf;
547 {
548 llvm::raw_string_ostream OS(Buf);
549 llvm::yaml::Output Yout(OS);
550 Relation Rel = R; // copy: Yout<< requires mutability.
551 Yout << Rel;
552 }
553 return Buf;
554}
555
556std::string toYAML(const Ref &R) {
557 std::string Buf;
558 {
559 llvm::raw_string_ostream OS(Buf);
560 llvm::yaml::Output Yout(OS);
561 Ref Reference = R; // copy: Yout<< requires mutability.
562 Yout << Reference;
563 }
564 return Buf;
565}
566
567} // namespace clangd
568} // namespace clang
569

source code of clang-tools-extra/clangd/index/YAMLSerialization.cpp