1//===--- Token.cpp - Tokens and token streams in the pseudoparser ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Token.h"
10#include "clang/Basic/LangOptions.h"
11#include "llvm/ADT/StringExtras.h"
12#include "llvm/Support/Format.h"
13#include "llvm/Support/FormatVariadic.h"
14
15namespace clang {
16namespace clangd {
17
18llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token &T) {
19 OS << llvm::formatv(Fmt: "{0} {1}:{2} ", Vals: clang::tok::getTokenName(Kind: T.Kind), Vals: T.Line,
20 Vals: T.Indent);
21 OS << '"';
22 llvm::printEscapedString(Name: T.text(), Out&: OS);
23 OS << '"';
24 if (T.Flags)
25 OS << llvm::format(Fmt: " flags=%x", Vals: T.Flags);
26 return OS;
27}
28
29llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const TokenStream &TS) {
30 OS << "Index Kind Line Text\n";
31 for (const auto &T : TS.tokens()) {
32 OS << llvm::format(Fmt: "%5d: %16s %4d:%-2d ", Vals: TS.index(T),
33 Vals: clang::tok::getTokenName(Kind: T.Kind), Vals: T.Line, Vals: T.Indent);
34 OS << '"';
35 llvm::printEscapedString(Name: T.text(), Out&: OS);
36 OS << '"';
37 if (T.Flags)
38 OS << llvm::format(Fmt: " flags=%x", Vals: T.Flags);
39 OS << '\n';
40 }
41 return OS;
42}
43
44llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token::Range &R) {
45 OS << llvm::formatv(Fmt: "[{0},{1})", Vals: R.Begin, Vals: R.End);
46 return OS;
47}
48
49TokenStream::TokenStream(std::shared_ptr<void> Payload)
50 : Payload(std::move(Payload)) {
51 Storage.emplace_back();
52 Storage.back().Kind = clang::tok::eof;
53}
54
55void TokenStream::finalize() {
56 assert(!isFinalized());
57 unsigned LastLine = Storage.back().Line;
58 Storage.emplace_back();
59 Storage.back().Kind = tok::eof;
60 Storage.back().Line = LastLine + 1;
61
62 Tokens = Storage;
63 Tokens = Tokens.drop_front().drop_back();
64}
65
66bool TokenStream::isFinalized() const {
67 assert(!Storage.empty() && Storage.front().Kind == tok::eof);
68 if (Storage.size() == 1)
69 return false;
70 return Storage.back().Kind == tok::eof;
71}
72
73void TokenStream::print(llvm::raw_ostream &OS) const {
74 bool FirstToken = true;
75 unsigned LastLine = -1;
76 StringRef LastText;
77 for (const auto &T : tokens()) {
78 StringRef Text = T.text();
79 if (FirstToken) {
80 FirstToken = false;
81 } else if (T.Line == LastLine) {
82 if (LastText.data() + LastText.size() != Text.data())
83 OS << ' ';
84 } else {
85 OS << '\n';
86 OS.indent(NumSpaces: T.Indent);
87 }
88 OS << Text;
89 LastLine = T.Line;
90 LastText = Text;
91 }
92 if (!FirstToken)
93 OS << '\n';
94}
95
96clang::LangOptions genericLangOpts(clang::Language Lang,
97 clang::LangStandard::Kind Standard) {
98 clang::LangOptions Opts;
99 std::vector<std::string> UnusedIncludes;
100 LangOptions::setLangDefaults(Opts, Lang, T: llvm::Triple(), Includes&: UnusedIncludes,
101 LangStd: Standard);
102
103 // Some options are "on by default", but e.g. at the driver level.
104 if (Opts.CPlusPlus)
105 Opts.CXXOperatorNames = true;
106 if (Opts.CPlusPlus20)
107 Opts.Coroutines = true;
108
109 // Some options are off by default, but define keywords we want to tolerate.
110 if (Opts.CPlusPlus)
111 Opts.MicrosoftExt = true; // kw__try, kw__finally
112 Opts.DeclSpecKeyword = true; // __declspec
113 Opts.WChar = true;
114
115 return Opts;
116}
117
118TokenStream stripComments(const TokenStream &Input) {
119 TokenStream Out(Input.getPayload());
120 for (const Token &T : Input.tokens()) {
121 if (T.Kind == tok::comment)
122 continue;
123 Out.push(T);
124 }
125 Out.finalize();
126 return Out;
127}
128
129} // namespace clangd
130} // namespace clang
131

source code of clang-tools-extra/clangd/support/Token.cpp