1//===--- Preamble.h - Reusing expensive parts of the AST ---------*- C++-*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The vast majority of code in a typical translation unit is in the headers
10// included at the top of the file.
11//
12// The preamble optimization says that we can parse this code once, and reuse
13// the result multiple times. The preamble is invalidated by changes to the
14// code in the preamble region, to the compile command, or to files on disk.
15//
16// This is the most important optimization in clangd: it allows operations like
17// code-completion to have sub-second latency. It is supported by the
18// PrecompiledPreamble functionality in clang, which wraps the techniques used
19// by PCH files, modules etc into a convenient interface.
20//
21//===----------------------------------------------------------------------===//
22#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_PREAMBLE_H
23#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_PREAMBLE_H
24
25#include "CollectMacros.h"
26#include "Compiler.h"
27#include "Diagnostics.h"
28#include "FS.h"
29#include "Headers.h"
30#include "ModulesBuilder.h"
31
32#include "clang-include-cleaner/Record.h"
33#include "support/Path.h"
34#include "clang/Basic/SourceManager.h"
35#include "clang/Basic/TargetOptions.h"
36#include "clang/Frontend/CompilerInvocation.h"
37#include "clang/Frontend/PrecompiledPreamble.h"
38#include "clang/Lex/Lexer.h"
39#include "clang/Tooling/CompilationDatabase.h"
40#include "llvm/ADT/ArrayRef.h"
41#include "llvm/ADT/StringRef.h"
42
43#include <cstddef>
44#include <functional>
45#include <memory>
46#include <string>
47#include <utility>
48#include <vector>
49
50namespace clang {
51namespace clangd {
52
53/// The captured AST context.
54/// Keeps necessary structs for an ASTContext and Preprocessor alive.
55/// This enables consuming them after context that produced the AST is gone.
56/// (e.g. indexing a preamble ast on a separate thread). ASTContext stored
57/// inside is still not thread-safe.
58
59struct CapturedASTCtx {
60public:
61 CapturedASTCtx(CompilerInstance &Clang)
62 : Invocation(Clang.getInvocationPtr()),
63 Diagnostics(Clang.getDiagnosticsPtr()), Target(Clang.getTargetPtr()),
64 AuxTarget(Clang.getAuxTarget()), FileMgr(Clang.getFileManagerPtr()),
65 SourceMgr(Clang.getSourceManagerPtr()), PP(Clang.getPreprocessorPtr()),
66 Context(Clang.getASTContextPtr()) {}
67
68 CapturedASTCtx(const CapturedASTCtx &) = delete;
69 CapturedASTCtx &operator=(const CapturedASTCtx &) = delete;
70 CapturedASTCtx(CapturedASTCtx &&) = default;
71 CapturedASTCtx &operator=(CapturedASTCtx &&) = default;
72
73 ASTContext &getASTContext() { return *Context; }
74 Preprocessor &getPreprocessor() { return *PP; }
75 CompilerInvocation &getCompilerInvocation() { return *Invocation; }
76 FileManager &getFileManager() { return *FileMgr; }
77 void setStatCache(std::shared_ptr<PreambleFileStatusCache> StatCache) {
78 this->StatCache = StatCache;
79 }
80
81private:
82 std::shared_ptr<CompilerInvocation> Invocation;
83 IntrusiveRefCntPtr<DiagnosticsEngine> Diagnostics;
84 IntrusiveRefCntPtr<TargetInfo> Target;
85 IntrusiveRefCntPtr<TargetInfo> AuxTarget;
86 IntrusiveRefCntPtr<FileManager> FileMgr;
87 IntrusiveRefCntPtr<SourceManager> SourceMgr;
88 std::shared_ptr<Preprocessor> PP;
89 IntrusiveRefCntPtr<ASTContext> Context;
90 std::shared_ptr<PreambleFileStatusCache> StatCache;
91};
92
93/// The parsed preamble and associated data.
94///
95/// As we must avoid re-parsing the preamble, any information that can only
96/// be obtained during parsing must be eagerly captured and stored here.
97struct PreambleData {
98 PreambleData(PrecompiledPreamble Preamble) : Preamble(std::move(Preamble)) {}
99
100 // Version of the ParseInputs this preamble was built from.
101 std::string Version;
102 tooling::CompileCommand CompileCommand;
103 // Target options used when building the preamble. Changes in target can cause
104 // crashes when deserializing preamble, this enables consumers to use the
105 // same target (without reparsing CompileCommand).
106 std::unique_ptr<TargetOptions> TargetOpts = nullptr;
107 PrecompiledPreamble Preamble;
108 std::vector<Diag> Diags;
109 // Processes like code completions and go-to-definitions will need #include
110 // information, and their compile action skips preamble range.
111 IncludeStructure Includes;
112 // Captures #include-mapping information in #included headers.
113 std::shared_ptr<const include_cleaner::PragmaIncludes> Pragmas;
114 // Information about required module files for this preamble.
115 std::unique_ptr<PrerequisiteModules> RequiredModules;
116 // Macros defined in the preamble section of the main file.
117 // Users care about headers vs main-file, not preamble vs non-preamble.
118 // These should be treated as main-file entities e.g. for code completion.
119 MainFileMacros Macros;
120 // Pragma marks defined in the preamble section of the main file.
121 std::vector<PragmaMark> Marks;
122 // Cache of FS operations performed when building the preamble.
123 // When reusing a preamble, this cache can be consumed to save IO.
124 std::shared_ptr<PreambleFileStatusCache> StatCache;
125 // Whether there was a (possibly-incomplete) include-guard on the main file.
126 // We need to propagate this information "by hand" to subsequent parses.
127 bool MainIsIncludeGuarded = false;
128};
129
130using PreambleParsedCallback =
131 std::function<void(CapturedASTCtx ASTCtx,
132 std::shared_ptr<const include_cleaner::PragmaIncludes>)>;
133
134/// Timings and statistics from the premble build. Unlike PreambleData, these
135/// do not need to be stored for later, but can be useful for logging, metrics,
136/// etc.
137struct PreambleBuildStats {
138 /// Total wall time it took to build preamble, in seconds.
139 double TotalBuildTime;
140 /// Time spent in filesystem operations during the build, in seconds.
141 double FileSystemTime;
142
143 /// Estimate of the memory used while building the preamble.
144 /// This memory has been released when buildPreamble returns.
145 /// For example, this includes the size of the in-memory AST (ASTContext).
146 size_t BuildSize;
147 /// The serialized size of the preamble.
148 /// This storage is needed while the preamble is used (but may be on disk).
149 size_t SerializedSize;
150};
151
152/// Build a preamble for the new inputs unless an old one can be reused.
153/// If \p PreambleCallback is set, it will be run on top of the AST while
154/// building the preamble.
155/// If Stats is not non-null, build statistics will be exported there.
156std::shared_ptr<const PreambleData>
157buildPreamble(PathRef FileName, CompilerInvocation CI,
158 const ParseInputs &Inputs, bool StoreInMemory,
159 PreambleParsedCallback PreambleCallback,
160 PreambleBuildStats *Stats = nullptr);
161
162/// Returns true if \p Preamble is reusable for \p Inputs. Note that it will
163/// return true when some missing headers are now available.
164/// FIXME: Should return more information about the delta between \p Preamble
165/// and \p Inputs, e.g. new headers.
166bool isPreambleCompatible(const PreambleData &Preamble,
167 const ParseInputs &Inputs, PathRef FileName,
168 const CompilerInvocation &CI);
169
170/// Stores information required to parse a TU using a (possibly stale) Baseline
171/// preamble. Later on this information can be injected into the main file by
172/// updating compiler invocation with \c apply. This injected section
173/// approximately reflects additions to the preamble in Modified contents, e.g.
174/// new include directives.
175class PreamblePatch {
176public:
177 enum class PatchType { MacroDirectives, All };
178 /// \p Preamble is used verbatim.
179 static PreamblePatch unmodified(const PreambleData &Preamble);
180 /// Builds a patch that contains new PP directives introduced to the preamble
181 /// section of \p Modified compared to \p Baseline.
182 /// FIXME: This only handles include directives, we should at least handle
183 /// define/undef.
184 static PreamblePatch createFullPatch(llvm::StringRef FileName,
185 const ParseInputs &Modified,
186 const PreambleData &Baseline);
187 static PreamblePatch createMacroPatch(llvm::StringRef FileName,
188 const ParseInputs &Modified,
189 const PreambleData &Baseline);
190 /// Returns the FileEntry for the preamble patch of MainFilePath in SM, if
191 /// any.
192 static OptionalFileEntryRef getPatchEntry(llvm::StringRef MainFilePath,
193 const SourceManager &SM);
194
195 /// Adjusts CI (which compiles the modified inputs) to be used with the
196 /// baseline preamble. This is done by inserting an artificial include to the
197 /// \p CI that contains new directives calculated in create.
198 void apply(CompilerInvocation &CI) const;
199
200 /// Returns #include directives from the \c Modified preamble that were
201 /// resolved using the \c Baseline preamble. This covers the new locations of
202 /// inclusions that were moved around, but not inclusions of new files. Those
203 /// will be recorded when parsing the main file: the includes in the injected
204 /// section will be resolved back to their spelled positions in the main file
205 /// using the presumed-location mechanism.
206 std::vector<Inclusion> preambleIncludes() const;
207
208 /// Returns preamble bounds for the Modified.
209 PreambleBounds modifiedBounds() const { return ModifiedBounds; }
210
211 /// Returns textual patch contents.
212 llvm::StringRef text() const { return PatchContents; }
213
214 /// Returns diag locations for Modified contents.
215 llvm::ArrayRef<Diag> patchedDiags() const { return PatchedDiags; }
216
217 static constexpr llvm::StringLiteral HeaderName = "__preamble_patch__.h";
218
219 llvm::ArrayRef<PragmaMark> marks() const;
220 const MainFileMacros &mainFileMacros() const;
221
222private:
223 static PreamblePatch create(llvm::StringRef FileName,
224 const ParseInputs &Modified,
225 const PreambleData &Baseline,
226 PatchType PatchType);
227
228 PreamblePatch() = default;
229 std::string PatchContents;
230 std::string PatchFileName;
231 // Includes that are present in both Baseline and Modified. Used for
232 // patching includes of baseline preamble.
233 std::vector<Inclusion> PreambleIncludes;
234 // Diags that were attached to a line preserved in Modified contents.
235 std::vector<Diag> PatchedDiags;
236 PreambleBounds ModifiedBounds = {0, false};
237 const PreambleData *Baseline = nullptr;
238 std::vector<PragmaMark> PatchedMarks;
239 MainFileMacros PatchedMacros;
240};
241
242} // namespace clangd
243} // namespace clang
244
245#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_PREAMBLE_H
246

source code of clang-tools-extra/clangd/Preamble.h