1	//===- unittest/Format/TokenAnnotatorTest.cpp - Formatting unit tests -----===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "clang/Format/Format.h"
10
11	#include "FormatTestUtils.h"
12	#include "TestLexer.h"
13	#include "gtest/gtest.h"
14
15	namespace clang {
16	namespace format {
17
18	// Not really the equality, but everything we need.
19	static bool operator==(const FormatToken &LHS,
20	const FormatToken &RHS) noexcept {
21	return LHS.Tok.getKind() == RHS.Tok.getKind() &&
22	LHS.getType() == RHS.getType();
23	}
24
25	namespace {
26
27	class TokenAnnotatorTest : public testing::Test {
28	protected:
29	TokenList annotate(StringRef Code,
30	const FormatStyle &Style = getLLVMStyle()) {
31	return TestLexer(Allocator, Buffers, Style).annotate(Code);
32	}
33	llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
34	std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
35	};
36
37	#define EXPECT_TOKEN_KIND(FormatTok, Kind) \
38	EXPECT_EQ((FormatTok)->Tok.getKind(), Kind) << *(FormatTok)
39	#define EXPECT_TOKEN_TYPE(FormatTok, Type) \
40	EXPECT_EQ((FormatTok)->getType(), Type) << *(FormatTok)
41	#define EXPECT_TOKEN_PRECEDENCE(FormatTok, Prec) \
42	EXPECT_EQ((FormatTok)->getPrecedence(), Prec) << *(FormatTok)
43	#define EXPECT_BRACE_KIND(FormatTok, Kind) \
44	EXPECT_EQ(FormatTok->getBlockKind(), Kind) << *(FormatTok)
45	#define EXPECT_SPLIT_PENALTY(FormatTok, Penalty) \
46	EXPECT_EQ(FormatTok->SplitPenalty, Penalty) << *(FormatTok)
47	#define EXPECT_TOKEN(FormatTok, Kind, Type) \
48	do { \
49	EXPECT_TOKEN_KIND(FormatTok, Kind); \
50	EXPECT_TOKEN_TYPE(FormatTok, Type); \
51	} while (false)
52
53	TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) {
54	auto Tokens = annotate(Code: "auto x = [](const decltype(x) &ptr) {};");
55	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
56	EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);
57	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);
58	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);
59	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
60	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
61
62	Tokens = annotate(Code: "auto x = [](const decltype(x) *ptr) {};");
63	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
64	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
65	EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
66
67	Tokens = annotate(Code: "#define lambda [](const decltype(x) &ptr) {}");
68	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
69	EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);
70	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);
71	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);
72	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
73	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
74
75	Tokens = annotate(Code: "#define lambda [](const decltype(x) *ptr) {}");
76	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
77	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
78	EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
79
80	Tokens = annotate(Code: "void f() {\n"
81	" while (p < a && *p == 'a')\n"
82	" p++;\n"
83	"}");
84	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
85	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
86	EXPECT_TOKEN(Tokens[11], tok::star, TT_UnaryOperator);
87
88	Tokens = annotate(Code: "case *x:");
89	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
90	EXPECT_TOKEN(Tokens[1], tok::star, TT_UnaryOperator);
91	Tokens = annotate(Code: "case &x:");
92	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
93	EXPECT_TOKEN(Tokens[1], tok::amp, TT_UnaryOperator);
94
95	Tokens = annotate(Code: "bool b = 3 == int{3} && true;");
96	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
97	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
98
99	Tokens = annotate(Code: "struct {\n"
100	"} *ptr;");
101	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
102	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
103	Tokens = annotate(Code: "union {\n"
104	"} *ptr;");
105	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
106	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
107	Tokens = annotate(Code: "class {\n"
108	"} *ptr;");
109	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
110	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
111
112	Tokens = annotate(Code: "struct {\n"
113	"} &&ptr = {};");
114	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
115	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
116	Tokens = annotate(Code: "union {\n"
117	"} &&ptr = {};");
118	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
119	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
120	Tokens = annotate(Code: "class {\n"
121	"} &&ptr = {};");
122	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
123	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
124	Tokens = annotate(Code: "int i = int{42} * 2;");
125	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
126	EXPECT_TOKEN(Tokens[7], tok::star, TT_BinaryOperator);
127
128	Tokens = annotate(Code: "delete[] *ptr;");
129	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
130	EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
131	Tokens = annotate(Code: "delete[] **ptr;");
132	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
133	EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
134	EXPECT_TOKEN(Tokens[4], tok::star, TT_UnaryOperator);
135	Tokens = annotate(Code: "delete[] *(ptr);");
136	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
137	EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
138
139	Tokens = annotate(Code: "void f() { void (*fnptr)(char* foo); }");
140	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
141	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
142	// FIXME: The star of a function pointer probably makes more sense as
143	// TT_PointerOrReference.
144	EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
145	EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
146
147	Tokens = annotate(Code: "void f() { void (*fnptr)(t* foo); }");
148	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
149	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
150	EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
151	EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
152
153	Tokens = annotate(Code: "int f3() { return sizeof(Foo&); }");
154	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
155	EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
156
157	Tokens = annotate(Code: "int f4() { return sizeof(Foo&&); }");
158	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
159	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
160
161	Tokens = annotate(Code: "void f5() { int f6(Foo&, Bar&); }");
162	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
163	EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
164	EXPECT_TOKEN(Tokens[12], tok::amp, TT_PointerOrReference);
165
166	Tokens = annotate(Code: "void f7() { int f8(Foo&&, Bar&&); }");
167	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
168	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
169	EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_PointerOrReference);
170
171	Tokens = annotate(Code: "Type1 &val1 = val2;");
172	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
173	EXPECT_TOKEN(Tokens[1], tok::amp, TT_PointerOrReference);
174
175	Tokens = annotate(Code: "Type1 *val1 = &val2;");
176	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
177	EXPECT_TOKEN(Tokens[1], tok::star, TT_PointerOrReference);
178	EXPECT_TOKEN(Tokens[4], tok::amp, TT_UnaryOperator);
179
180	Tokens = annotate(Code: "val1 & val2;");
181	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
182	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
183
184	Tokens = annotate(Code: "val1 & val2.member;");
185	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
186	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
187
188	Tokens = annotate(Code: "val1 & val2.*member;");
189	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
190	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
191
192	Tokens = annotate(Code: "val1.*member & val2;");
193	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
194	EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
195
196	Tokens = annotate(Code: "val1 & val2->*member;");
197	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
198	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
199
200	Tokens = annotate(Code: "val1->member & val2;");
201	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
202	EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
203
204	Tokens = annotate(Code: "val1 & val2 & val3;");
205	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
206	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
207	EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
208
209	Tokens = annotate(Code: "val1 & val2 // comment\n"
210	" & val3;");
211	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
212	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
213	EXPECT_TOKEN(Tokens[4], tok::amp, TT_BinaryOperator);
214
215	Tokens =
216	annotate(Code: "val1 & val2.member & val3.member() & val4 & val5->member;");
217	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
218	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
219	EXPECT_TOKEN(Tokens[5], tok::amp, TT_BinaryOperator);
220	EXPECT_TOKEN(Tokens[11], tok::amp, TT_BinaryOperator);
221	EXPECT_TOKEN(Tokens[13], tok::amp, TT_BinaryOperator);
222
223	Tokens = annotate(Code: "class c {\n"
224	" void func(type &a) { a & member; }\n"
225	" anotherType &member;\n"
226	"}");
227	ASSERT_EQ(Tokens.size(), 22u) << Tokens;
228	EXPECT_TOKEN(Tokens[7], tok::amp, TT_PointerOrReference);
229	EXPECT_TOKEN(Tokens[12], tok::amp, TT_BinaryOperator);
230	EXPECT_TOKEN(Tokens[17], tok::amp, TT_PointerOrReference);
231
232	Tokens = annotate(Code: "struct S {\n"
233	" auto Mem = C & D;\n"
234	"}");
235	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
236	EXPECT_TOKEN(Tokens[7], tok::amp, TT_BinaryOperator);
237
238	Tokens =
239	annotate(Code: "template <typename T> void swap() noexcept(Bar<T> && Foo<T>);");
240	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
241	EXPECT_TOKEN(Tokens[15], tok::ampamp, TT_BinaryOperator);
242
243	Tokens = annotate(Code: "template <typename T> struct S {\n"
244	" explicit(Bar<T> && Foo<T>) S(const S &);\n"
245	"};");
246	ASSERT_EQ(Tokens.size(), 30u) << Tokens;
247	EXPECT_TOKEN(Tokens[14], tok::ampamp, TT_BinaryOperator);
248
249	Tokens = annotate(Code: "template <bool B = C && D> struct S {};");
250	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
251	EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_BinaryOperator);
252
253	Tokens = annotate(Code: "template <typename T, bool B = C && D> struct S {};");
254	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
255	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
256
257	Tokens = annotate(Code: "template <typename T, typename U = T&&> struct S {};");
258	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
259	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
260
261	Tokens = annotate(Code: "template <typename T = int (*)(int)> struct S {};");
262	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
263	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
264	EXPECT_TOKEN(Tokens[7], tok::star, TT_PointerOrReference);
265
266	Tokens = annotate(Code: "Foo<A && B> a = {};");
267	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
268	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator);
269
270	Tokens = annotate(Code: "Foo<A &&> a = {};");
271	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
272	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
273
274	Tokens = annotate(Code: "template <typename T>\n"
275	"enable_if_t<is_integral_v<T>, bool> // comment\n"
276	"operator~(T &a);");
277	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
278	EXPECT_TOKEN(Tokens[19], tok::amp, TT_PointerOrReference);
279
280	Tokens = annotate(Code: "template <enable_if_t<foo && !bar>* = nullptr> void f();");
281	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
282	EXPECT_TOKEN(Tokens[5], tok::ampamp, TT_BinaryOperator);
283
284	Tokens =
285	annotate(Code: "auto foo() noexcept(noexcept(bar()) && "
286	"trait<std::decay_t<decltype(bar())>> && noexcept(baz())) {}");
287	ASSERT_EQ(Tokens.size(), 38u) << Tokens;
288	EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_BinaryOperator);
289	EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator);
290
291	Tokens = annotate(Code: "foo = *i < *j && *j > *k;");
292	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
293	EXPECT_TOKEN(Tokens[4], tok::less, TT_BinaryOperator);
294	EXPECT_TOKEN(Tokens[7], tok::ampamp, TT_BinaryOperator);
295	EXPECT_TOKEN(Tokens[10], tok::greater, TT_BinaryOperator);
296
297	Tokens = annotate(Code: "if (Foo *foo; bar)");
298	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
299	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
300
301	Tokens = annotate(Code: "if (Foo **foo(); bar)");
302	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
303	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
304	EXPECT_TOKEN(Tokens[4], tok::star, TT_PointerOrReference);
305
306	Tokens = annotate(Code: "if (Foo *&foo{a}; bar)");
307	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
308	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
309	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
310
311	FormatStyle Style = getLLVMStyle();
312	Style.TypeNames.push_back(x: "MYI");
313	Tokens = annotate(Code: "if (MYI *p{nullptr})", Style);
314	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
315	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TypeName);
316	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
317
318	Style.TypeNames.push_back(x: "Class");
319	Tokens = annotate(Code: "if (Class *obj {getObj()})", Style);
320	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
321	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TypeName);
322	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
323
324	Tokens = annotate(Code: "class Foo {\n"
325	" void operator<() {}\n"
326	" Foo &f;\n"
327	"};");
328	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
329	EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);
330	EXPECT_TOKEN(Tokens[5], tok::less, TT_OverloadedOperator);
331	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen);
332	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);
333	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
334
335	Tokens = annotate(Code: "if (new && num) {\n"
336	" new = 1;\n"
337	"}\n"
338	"if (!delete && num) {\n"
339	" delete = 1;\n"
340	"}");
341	ASSERT_EQ(Tokens.size(), 26u) << Tokens;
342	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator);
343	EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator);
344
345	Tokens = annotate(Code: "#define FOO \\\n"
346	" void foo() { f(a * b); }");
347	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
348	EXPECT_TOKEN(Tokens[11], tok::star, TT_BinaryOperator);
349
350	Tokens = annotate(Code: "for (int i; Foo *&foo : foos)");
351	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
352	EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);
353	EXPECT_TOKEN(Tokens[7], tok::amp, TT_PointerOrReference);
354	EXPECT_TOKEN(Tokens[9], tok::colon, TT_RangeBasedForLoopColon);
355
356	Tokens = annotate(Code: "#define FOO auto Foo = [] { f(a * b); };");
357	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
358	EXPECT_TOKEN(Tokens[12], tok::star, TT_BinaryOperator);
359
360	Tokens = annotate(Code: "namespace {\n"
361	"#define FOO(x) void foo(a##x *b);\n"
362	"}");
363	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
364	EXPECT_TOKEN(Tokens[14], tok::star, TT_PointerOrReference);
365
366	Tokens = annotate(Code: "#define foo(x) _Generic(x, bar *: 1, default: 0)");
367	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
368	EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
369
370	Tokens = annotate(Code: "Thingy kConfig = {\n"
371	" 1,\n"
372	" (uint16_t)(kScale * height_pixels),\n"
373	"};");
374	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
375	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_CastRParen);
376	EXPECT_TOKEN(Tokens[11], tok::star, TT_BinaryOperator);
377
378	Tokens = annotate(Code: "template <typename T>\n"
379	"concept C = requires(T a, T b) { a && b; };");
380	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
381	EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_RequiresExpressionLBrace);
382	EXPECT_TOKEN(Tokens[18], tok::ampamp, TT_BinaryOperator);
383
384	Tokens = annotate(Code: "template <typename T, typename V>\n"
385	"concept CheckMultiplicableBy = requires(T a, V b) {\n"
386	" { a * b } -> std::same_as<T>;\n"
387	"};");
388	ASSERT_EQ(Tokens.size(), 36u) << Tokens;
389	EXPECT_TOKEN(Tokens[19], tok::l_brace, TT_RequiresExpressionLBrace);
390	EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_CompoundRequirementLBrace);
391	EXPECT_TOKEN(Tokens[22], tok::star, TT_BinaryOperator);
392
393	Tokens = annotate(Code: "bool foo = requires { static_cast<Foo &&>(1); };");
394	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
395	EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_PointerOrReference);
396
397	Tokens = annotate(Code: "return s.operator int *();");
398	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
399	// Not TT_FunctionDeclarationName.
400	EXPECT_TOKEN(Tokens[3], tok::kw_operator, TT_Unknown);
401	EXPECT_TOKEN(Tokens[5], tok::star, TT_PointerOrReference);
402	// Not TT_FunctionDeclarationLParen.
403	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_Unknown);
404
405	Tokens = annotate(Code: "B &b = x.operator B &();");
406	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
407	EXPECT_TOKEN(Tokens[8], tok::amp, TT_PointerOrReference);
408
409	Tokens = annotate(Code: "int8_t *a = MacroCall(int8_t, width * height * length);");
410	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
411	EXPECT_TOKEN(Tokens[9], tok::star, TT_BinaryOperator);
412	EXPECT_TOKEN(Tokens[11], tok::star, TT_BinaryOperator);
413
414	Tokens = annotate(Code: "float foo[3] = {M * H * H, H * M * H, H * H * M};");
415	ASSERT_EQ(Tokens.size(), 27u) << Tokens;
416	EXPECT_TOKEN(Tokens[16], tok::star, TT_BinaryOperator);
417	EXPECT_TOKEN(Tokens[22], tok::star, TT_BinaryOperator);
418
419	Tokens = annotate(Code: "NSError *__autoreleasing *foo;",
420	Style: getLLVMStyle(Language: FormatStyle::LK_ObjC));
421	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
422	EXPECT_TOKEN(Tokens[1], tok::star, TT_PointerOrReference);
423	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
424	}
425
426	TEST_F(TokenAnnotatorTest, UnderstandsUsesOfPlusAndMinus) {
427	auto Tokens = annotate(Code: "x - 0");
428	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
429	EXPECT_TOKEN(Tokens[1], tok::minus, TT_BinaryOperator);
430	Tokens = annotate(Code: "0 + 0");
431	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
432	EXPECT_TOKEN(Tokens[1], tok::plus, TT_BinaryOperator);
433	Tokens = annotate(Code: "x + +0");
434	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
435	EXPECT_TOKEN(Tokens[2], tok::plus, TT_UnaryOperator);
436	Tokens = annotate(Code: "x ? -0 : +0");
437	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
438	EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
439	EXPECT_TOKEN(Tokens[5], tok::plus, TT_UnaryOperator);
440	Tokens = annotate(Code: "(-0)");
441	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
442	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
443	Tokens = annotate(Code: "0, -0");
444	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
445	EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
446	Tokens = annotate(Code: "for (; -1;) {\n}");
447	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
448	EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
449	Tokens = annotate(Code: "x = -1;");
450	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
451	EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
452	Tokens = annotate(Code: "x[-1]");
453	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
454	EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
455	Tokens = annotate(Code: "x = {-1};");
456	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
457	EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
458	Tokens = annotate(Code: "case -x:");
459	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
460	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
461	Tokens = annotate(Code: "co_await -x;");
462	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
463	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
464	Tokens = annotate(Code: "co_return -x;");
465	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
466	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
467	Tokens = annotate(Code: "co_yield -x;");
468	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
469	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
470	Tokens = annotate(Code: "delete -x;");
471	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
472	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
473	Tokens = annotate(Code: "return -x;");
474	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
475	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
476	Tokens = annotate(Code: "throw -x;");
477	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
478	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
479	Tokens = annotate(Code: "sizeof -x");
480	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
481	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
482	Tokens = annotate(Code: "co_await +x;");
483	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
484	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
485	Tokens = annotate(Code: "co_return +x;");
486	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
487	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
488	Tokens = annotate(Code: "co_yield +x;");
489	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
490	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
491	Tokens = annotate(Code: "delete +x;");
492	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
493	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
494	Tokens = annotate(Code: "return +x;");
495	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
496	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
497	Tokens = annotate(Code: "throw +x;");
498	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
499	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
500	Tokens = annotate(Code: "sizeof +x");
501	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
502	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
503	Tokens = annotate(Code: "(int)-x");
504	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
505	EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
506	Tokens = annotate(Code: "(-x)");
507	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
508	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
509	Tokens = annotate(Code: "!+x");
510	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
511	EXPECT_TOKEN(Tokens[0], tok::exclaim, TT_UnaryOperator);
512	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
513	}
514
515	TEST_F(TokenAnnotatorTest, UnderstandsClasses) {
516	auto Tokens = annotate(Code: "class C {};");
517	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
518	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
519	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);
520	EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_ClassRBrace);
521
522	Tokens = annotate(Code: "const class C {} c;");
523	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
524	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);
525	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_ClassLBrace);
526	EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_ClassRBrace);
527
528	Tokens = annotate(Code: "const class {} c;");
529	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
530	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);
531	EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_ClassRBrace);
532
533	Tokens = annotate(Code: "class [[deprecated(\"\")]] C { int i; };");
534	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
535	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_ClassHeadName);
536	EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_ClassLBrace);
537	EXPECT_TOKEN(Tokens[14], tok::r_brace, TT_ClassRBrace);
538	}
539
540	TEST_F(TokenAnnotatorTest, UnderstandsStructs) {
541	auto Tokens = annotate(Code: "struct S {};");
542	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
543	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
544	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_StructLBrace);
545	EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_StructRBrace);
546
547	Tokens = annotate(Code: "struct macro(a) S {};");
548	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
549	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_ClassHeadName);
550	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_StructLBrace);
551	EXPECT_TOKEN(Tokens[7], tok::r_brace, TT_StructRBrace);
552
553	Tokens = annotate(Code: "struct EXPORT_MACRO [[nodiscard]] C { int i; };");
554	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
555	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_ClassHeadName);
556	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_StructLBrace);
557	EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_StructRBrace);
558
559	Tokens = annotate(Code: "struct [[deprecated]] [[nodiscard]] C { int i; };");
560	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
561	EXPECT_TOKEN(Tokens[11], tok::identifier, TT_ClassHeadName);
562	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_StructLBrace);
563	EXPECT_TOKEN(Tokens[16], tok::r_brace, TT_StructRBrace);
564
565	Tokens = annotate(Code: "struct macro(a) S {\n"
566	" void f(T &t);\n"
567	"};");
568	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
569	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_ClassHeadName);
570	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_StructLBrace);
571	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
572	EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_StructRBrace);
573
574	Tokens = annotate(Code: "template <typename T> struct S<const T[N]> {};");
575	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
576	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);
577	EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);
578	EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);
579	EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
580	EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_StructLBrace);
581	EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_StructRBrace);
582
583	Tokens = annotate(Code: "template <typename T> struct S<T const[N]> {};");
584	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
585	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);
586	EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);
587	EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);
588	EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
589	EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_StructLBrace);
590	EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_StructRBrace);
591
592	Tokens = annotate(Code: "template <typename T, unsigned n> struct S<T const[n]> {\n"
593	" void f(T const (&a)[n]);\n"
594	"};");
595	ASSERT_EQ(Tokens.size(), 35u) << Tokens;
596	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_ClassHeadName);
597	EXPECT_TOKEN(Tokens[10], tok::less, TT_TemplateOpener);
598	EXPECT_TOKEN(Tokens[13], tok::l_square, TT_ArraySubscriptLSquare);
599	EXPECT_TOKEN(Tokens[16], tok::greater, TT_TemplateCloser);
600	EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_StructLBrace);
601	EXPECT_TOKEN(Tokens[23], tok::l_paren, TT_FunctionTypeLParen);
602	EXPECT_TOKEN(Tokens[24], tok::amp, TT_UnaryOperator);
603	EXPECT_TOKEN(Tokens[27], tok::l_square, TT_ArraySubscriptLSquare);
604	EXPECT_TOKEN(Tokens[32], tok::r_brace, TT_StructRBrace);
605
606	Tokens = annotate(Code: "template <typename T, enum E e> struct S {};");
607	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
608	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_ClassHeadName);
609	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_StructLBrace);
610
611	Tokens = annotate(
612	Code: "template <> struct __declspec(foo) Op<Bar *> : OpImpl<Bar *> {};");
613	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
614	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_AttributeLParen);
615	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_AttributeRParen);
616	EXPECT_TOKEN(Tokens[8], tok::identifier, TT_ClassHeadName);
617	EXPECT_TOKEN(Tokens[13], tok::colon, TT_InheritanceColon);
618	EXPECT_TOKEN(Tokens[19], tok::l_brace, TT_StructLBrace);
619	EXPECT_TOKEN(Tokens[20], tok::r_brace, TT_StructRBrace);
620
621	Tokens = annotate(Code: "class Outer {\n"
622	" struct Inner final : Base {};\n"
623	"};");
624	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
625	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_Unknown); // Not TT_StartOfName
626	EXPECT_TOKEN(Tokens[6], tok::colon, TT_InheritanceColon);
627
628	constexpr StringRef Code{"struct EXPORT StructName {};"};
629
630	Tokens = annotate(Code);
631	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
632	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);
633	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_StructLBrace);
634	EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_StructRBrace);
635
636	auto Style = getLLVMStyle();
637	Style.AttributeMacros.push_back(x: "EXPORT");
638	Tokens = annotate(Code, Style);
639	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
640	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_AttributeMacro);
641	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);
642	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_StructLBrace);
643	EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_StructRBrace);
644	}
645
646	TEST_F(TokenAnnotatorTest, UnderstandsUnions) {
647	auto Tokens = annotate(Code: "union U {};");
648	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
649	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
650	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
651	EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_UnionRBrace);
652
653	Tokens = annotate(Code: "union U { void f() { return; } };");
654	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
655	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
656	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
657	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
658	EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_UnionRBrace);
659	}
660
661	TEST_F(TokenAnnotatorTest, UnderstandsEnums) {
662	auto Tokens = annotate(Code: "enum E {};");
663	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
664	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_EnumLBrace);
665	EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_EnumRBrace);
666	}
667
668	TEST_F(TokenAnnotatorTest, UnderstandsDefaultedAndDeletedFunctions) {
669	auto Tokens = annotate(Code: "auto operator<=>(const T &) const & = default;");
670	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
671	EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
672
673	Tokens = annotate(Code: "template <typename T> void F(T) && = delete;");
674	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
675	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
676	}
677
678	TEST_F(TokenAnnotatorTest, UnderstandsVariables) {
679	auto Tokens =
680	annotate(Code: "inline bool var = is_integral_v<int> && is_signed_v<int>;");
681	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
682	EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_BinaryOperator);
683	}
684
685	TEST_F(TokenAnnotatorTest, UnderstandsVariableTemplates) {
686	auto Tokens =
687	annotate(Code: "template <typename T> "
688	"inline bool var = is_integral_v<int> && is_signed_v<int>;");
689	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
690	EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
691	}
692
693	TEST_F(TokenAnnotatorTest, UnderstandsTemplatesInMacros) {
694	auto Tokens =
695	annotate(Code: "#define FOO(typeName) \\\n"
696	" { #typeName, foo<FooType>(new foo<realClass>(#typeName)) }");
697	ASSERT_EQ(Tokens.size(), 27u) << Tokens;
698	EXPECT_TOKEN(Tokens[11], tok::less, TT_TemplateOpener);
699	EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
700	EXPECT_TOKEN(Tokens[17], tok::less, TT_TemplateOpener);
701	EXPECT_TOKEN(Tokens[19], tok::greater, TT_TemplateCloser);
702	}
703
704	TEST_F(TokenAnnotatorTest, UnderstandsGreaterAfterTemplateCloser) {
705	auto Tokens = annotate(Code: "if (std::tuple_size_v<T> > 0)");
706	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
707	EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);
708	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
709	EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
710	}
711
712	TEST_F(TokenAnnotatorTest, UnderstandsTernaryInTemplate) {
713	// IsExpression = false
714	auto Tokens = annotate(Code: "foo<true ? 1 : 2>();");
715	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
716	EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
717	EXPECT_TOKEN(Tokens[3], tok::question, TT_ConditionalExpr);
718	EXPECT_TOKEN(Tokens[5], tok::colon, TT_ConditionalExpr);
719	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
720
721	// IsExpression = true
722
723	Tokens = annotate(Code: "return foo<true ? 1 : 2>();");
724	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
725	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
726	EXPECT_TOKEN(Tokens[4], tok::question, TT_ConditionalExpr);
727	EXPECT_TOKEN(Tokens[6], tok::colon, TT_ConditionalExpr);
728	EXPECT_TOKEN(Tokens[8], tok::greater, TT_TemplateCloser);
729
730	Tokens = annotate(Code: "return foo<true ? 1 : 2>{};");
731	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
732	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
733	EXPECT_TOKEN(Tokens[4], tok::question, TT_ConditionalExpr);
734	EXPECT_TOKEN(Tokens[6], tok::colon, TT_ConditionalExpr);
735	EXPECT_TOKEN(Tokens[8], tok::greater, TT_TemplateCloser);
736	}
737
738	TEST_F(TokenAnnotatorTest, UnderstandsNonTemplateAngleBrackets) {
739	auto Tokens = annotate(Code: "return a < b && c > d;");
740	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
741	EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
742	EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);
743
744	Tokens = annotate(Code: "a < 0 ? b : a > 0 ? c : d;");
745	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
746	EXPECT_TOKEN(Tokens[1], tok::less, TT_BinaryOperator);
747	EXPECT_TOKEN(Tokens[7], tok::greater, TT_BinaryOperator);
748
749	Tokens = annotate(Code: "return A < B ? true : A > B;");
750	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
751	EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
752	EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
753
754	Tokens = annotate(Code: "return A < B ? true : A > B ? false : false;");
755	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
756	EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
757	EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
758
759	Tokens = annotate(Code: "return checklower ? a < b : a > b;");
760	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
761	EXPECT_TOKEN(Tokens[4], tok::less, TT_BinaryOperator);
762	EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
763
764	Tokens = annotate(Code: "return A < B != A > B;");
765	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
766	// FIXME:
767	// EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
768	// EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);
769
770	Tokens = annotate(Code: "ratio{-1, 2} < ratio{-1, 3} == -1 / 3 > -1 / 2;");
771	ASSERT_EQ(Tokens.size(), 27u) << Tokens;
772	EXPECT_TOKEN(Tokens[7], tok::less, TT_BinaryOperator);
773	EXPECT_TOKEN(Tokens[20], tok::greater, TT_BinaryOperator);
774
775	Tokens = annotate(Code: "bool foo = a < b && (c * d) > e;");
776	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
777	EXPECT_TOKEN(Tokens[4], tok::less, TT_BinaryOperator);
778	EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_BinaryOperator);
779	EXPECT_TOKEN(Tokens[9], tok::star, TT_BinaryOperator);
780	EXPECT_TOKEN(Tokens[12], tok::greater, TT_BinaryOperator);
781	}
782
783	TEST_F(TokenAnnotatorTest, UnderstandsTemplateTemplateParameters) {
784	auto Tokens = annotate(Code: "template <template <typename...> typename X,\n"
785	" template <typename...> class Y,\n"
786	" typename... T>\n"
787	"class A {};");
788	ASSERT_EQ(Tokens.size(), 28u) << Tokens;
789	EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
790	EXPECT_TOKEN(Tokens[3], tok::less, TT_TemplateOpener);
791	EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
792	EXPECT_FALSE(Tokens[6]->ClosesTemplateDeclaration);
793	EXPECT_TOKEN(Tokens[11], tok::less, TT_TemplateOpener);
794	EXPECT_TOKEN(Tokens[14], tok::greater, TT_TemplateCloser);
795	EXPECT_FALSE(Tokens[14]->ClosesTemplateDeclaration);
796	EXPECT_TOKEN(Tokens[16], tok::identifier, TT_Unknown);
797	EXPECT_TOKEN(Tokens[21], tok::greater, TT_TemplateCloser);
798	EXPECT_TRUE(Tokens[21]->ClosesTemplateDeclaration);
799	EXPECT_TOKEN(Tokens[23], tok::identifier, TT_ClassHeadName);
800	}
801
802	TEST_F(TokenAnnotatorTest, UnderstandsWhitespaceSensitiveMacros) {
803	FormatStyle Style = getLLVMStyle();
804	Style.WhitespaceSensitiveMacros.push_back(x: "FOO");
805
806	auto Tokens = annotate(Code: "FOO(1+2 )", Style);
807	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
808	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);
809
810	Tokens = annotate(Code: "FOO(a:b:c)", Style);
811	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
812	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);
813	}
814
815	TEST_F(TokenAnnotatorTest, UnderstandsDelete) {
816	auto Tokens = annotate(Code: "delete (void *)p;");
817	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
818	EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);
819
820	Tokens = annotate(Code: "delete[] (void *)p;");
821	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
822	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_CastRParen);
823
824	Tokens = annotate(Code: "delete[] /*comment*/ (void *)p;");
825	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
826	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
827
828	Tokens = annotate(Code: "delete[/*comment*/] (void *)p;");
829	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
830	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
831
832	Tokens = annotate(Code: "delete/*comment*/[] (void *)p;");
833	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
834	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
835	}
836
837	TEST_F(TokenAnnotatorTest, UnderstandsCasts) {
838	auto Tokens = annotate(Code: "(void)p;");
839	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
840	EXPECT_TOKEN(Tokens[2], tok::r_paren, TT_CastRParen);
841
842	Tokens = annotate(Code: "(uint32_t)&&label;");
843	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
844	EXPECT_TOKEN(Tokens[2], tok::r_paren, TT_CastRParen);
845	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_UnaryOperator);
846	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);
847
848	Tokens = annotate(Code: "auto x = (Foo)p;");
849	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
850	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_CastRParen);
851
852	Tokens = annotate(Code: "(std::vector<int>)p;");
853	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
854	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
855
856	Tokens = annotate(Code: "return (Foo)p;");
857	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
858	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);
859
860	Tokens = annotate(Code: "throw (Foo)p;");
861	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
862	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);
863
864	Tokens = annotate(Code: "#define FOO(x) (((uint64_t)(x) * BAR) / 100)");
865	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
866	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);
867	EXPECT_TOKEN(Tokens[13], tok::r_paren, TT_Unknown);
868	EXPECT_TOKEN(Tokens[14], tok::star, TT_BinaryOperator);
869
870	Tokens = annotate(Code: "#define foo(i) ((i) - bar)");
871	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
872	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_Unknown);
873	EXPECT_TOKEN(Tokens[10], tok::minus, TT_BinaryOperator);
874
875	Tokens = annotate(Code: "return (Foo) & 10;");
876	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
877	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_Unknown);
878	EXPECT_TOKEN(Tokens[4], tok::amp, TT_BinaryOperator);
879
880	Tokens = annotate(Code: "return (struct foo){};");
881	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
882	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
883	EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);
884
885	Tokens = annotate(Code: "#define FOO(bar) foo((uint64_t)&bar)");
886	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
887	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);
888	EXPECT_TOKEN(Tokens[11], tok::amp, TT_UnaryOperator);
889
890	Tokens = annotate(Code: "#define FOO(bar) foo((Foo) & bar)");
891	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
892	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_Unknown);
893	EXPECT_TOKEN(Tokens[11], tok::amp, TT_BinaryOperator);
894
895	Tokens = annotate(Code: "func((void (*)())&a);");
896	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
897	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionTypeLParen);
898	EXPECT_TOKEN(Tokens[5], tok::star, TT_PointerOrReference);
899	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_CastRParen);
900	EXPECT_TOKEN(Tokens[10], tok::amp, TT_UnaryOperator);
901
902	Tokens = annotate(Code: "int result = ((int)a) - b;");
903	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
904	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_CastRParen);
905	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
906	EXPECT_TOKEN(Tokens[9], tok::minus, TT_BinaryOperator);
907
908	Tokens = annotate(Code: "return (double)(foo(30)) - 15;");
909	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
910	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);
911	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_Unknown);
912	EXPECT_TOKEN(Tokens[10], tok::minus, TT_BinaryOperator);
913
914	Tokens = annotate(Code: "return (::Type)(1 + 2);");
915	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
916	EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);
917
918	Tokens = annotate(Code: "return (Namespace::Class)(1 + 2);");
919	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
920	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_CastRParen);
921
922	Tokens = annotate(Code: "return (Foo (*)(void *, Bar, ...))&foo;");
923	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
924	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionTypeLParen);
925	EXPECT_TOKEN(Tokens[14], tok::r_paren, TT_CastRParen);
926	EXPECT_TOKEN(Tokens[15], tok::amp, TT_UnaryOperator);
927
928	Tokens = annotate(Code: "return (Foo (Bar::*)())&Bar::foo;");
929	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
930	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionTypeLParen);
931	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);
932	EXPECT_TOKEN(Tokens[11], tok::amp, TT_UnaryOperator);
933
934	auto Style = getLLVMStyle();
935	Style.TypeNames.push_back(x: "Foo");
936	Tokens = annotate(Code: "#define FOO(bar) foo((Foo)&bar)", Style);
937	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
938	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_TypeName);
939	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);
940	EXPECT_TOKEN(Tokens[11], tok::amp, TT_UnaryOperator);
941	}
942
943	TEST_F(TokenAnnotatorTest, UnderstandsDynamicExceptionSpecifier) {
944	auto Tokens = annotate(Code: "void f() throw(int);");
945	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
946	EXPECT_TOKEN(Tokens[4], tok::kw_throw, TT_Unknown);
947	}
948
949	TEST_F(TokenAnnotatorTest, UnderstandsFunctionRefQualifiers) {
950	auto Tokens = annotate(Code: "void f() &;");
951	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
952	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
953
954	Tokens = annotate(Code: "void operator=(T) &&;");
955	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
956	EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference);
957
958	Tokens = annotate(Code: "template <typename T> void f() &;");
959	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
960	EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
961
962	Tokens = annotate(Code: "template <typename T> void operator=(T) &;");
963	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
964	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
965	}
966
967	TEST_F(TokenAnnotatorTest, UnderstandsOverloadedOperators) {
968	auto Tokens = annotate(Code: "x.operator+()");
969	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
970	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
971	EXPECT_TOKEN(Tokens[3], tok::plus, TT_OverloadedOperator);
972	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
973	Tokens = annotate(Code: "x.operator=()");
974	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
975	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
976	EXPECT_TOKEN(Tokens[3], tok::equal, TT_OverloadedOperator);
977	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
978	Tokens = annotate(Code: "x.operator+=()");
979	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
980	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
981	EXPECT_TOKEN(Tokens[3], tok::plusequal, TT_OverloadedOperator);
982	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
983	Tokens = annotate(Code: "x.operator,()");
984	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
985	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
986	EXPECT_TOKEN(Tokens[3], tok::comma, TT_OverloadedOperator);
987	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
988	Tokens = annotate(Code: "x.operator()()");
989	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
990	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
991	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperator);
992	EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_OverloadedOperator);
993	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
994	Tokens = annotate(Code: "x.operator[]()");
995	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
996	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
997	// EXPECT_TOKEN(Tokens[3], tok::l_square, TT_OverloadedOperator);
998	// EXPECT_TOKEN(Tokens[4], tok::r_square, TT_OverloadedOperator);
999	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
1000	Tokens = annotate(Code: "x.operator\"\"_a()");
1001	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1002	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
1003	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
1004	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
1005	Tokens = annotate(Code: "x.operator\"\" _a()");
1006	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1007	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
1008	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
1009	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
1010	Tokens = annotate(Code: "x.operator\"\"if()");
1011	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1012	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
1013	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
1014	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
1015	Tokens = annotate(Code: "x.operator\"\"s()");
1016	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1017	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
1018	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
1019	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
1020	Tokens = annotate(Code: "x.operator\"\" s()");
1021	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1022	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
1023	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
1024	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
1025
1026	Tokens = annotate(Code: "int operator+(int);");
1027	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1028	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
1029	EXPECT_TOKEN(Tokens[2], tok::plus, TT_OverloadedOperator);
1030	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
1031	Tokens = annotate(Code: "auto operator=(T&) {}");
1032	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1033	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
1034	EXPECT_TOKEN(Tokens[2], tok::equal, TT_OverloadedOperator);
1035	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
1036	Tokens = annotate(Code: "auto operator()() {}");
1037	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1038	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
1039	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_OverloadedOperator);
1040	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_OverloadedOperator);
1041	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
1042
1043	Tokens = annotate(Code: "class Foo {\n"
1044	" int operator+(a* b);\n"
1045	"}");
1046	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1047	EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);
1048	EXPECT_TOKEN(Tokens[5], tok::plus, TT_OverloadedOperator);
1049	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen);
1050	EXPECT_TOKEN(Tokens[8], tok::star, TT_PointerOrReference);
1051
1052	Tokens = annotate(Code: "class Foo {\n"
1053	" int c = operator+(a * b);\n"
1054	"}");
1055	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1056	EXPECT_TOKEN(Tokens[6], tok::kw_operator, TT_Unknown);
1057	EXPECT_TOKEN(Tokens[7], tok::plus, TT_OverloadedOperator);
1058	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_OverloadedOperatorLParen);
1059	EXPECT_TOKEN(Tokens[10], tok::star, TT_BinaryOperator);
1060
1061	Tokens = annotate(Code: "void foo() {\n"
1062	" operator+(a * b);\n"
1063	"}");
1064	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1065	EXPECT_TOKEN(Tokens[5], tok::kw_operator, TT_Unknown);
1066	EXPECT_TOKEN(Tokens[6], tok::plus, TT_OverloadedOperator);
1067	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperatorLParen);
1068	EXPECT_TOKEN(Tokens[9], tok::star, TT_BinaryOperator);
1069
1070	Tokens = annotate(Code: "return operator+(a * b, c & d) + operator+(a && b && c);");
1071	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
1072	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_Unknown);
1073	EXPECT_TOKEN(Tokens[2], tok::plus, TT_OverloadedOperator);
1074	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
1075	EXPECT_TOKEN(Tokens[5], tok::star, TT_BinaryOperator);
1076	EXPECT_TOKEN(Tokens[9], tok::amp, TT_BinaryOperator);
1077	EXPECT_TOKEN(Tokens[13], tok::kw_operator, TT_Unknown);
1078	EXPECT_TOKEN(Tokens[14], tok::plus, TT_OverloadedOperator);
1079	EXPECT_TOKEN(Tokens[15], tok::l_paren, TT_OverloadedOperatorLParen);
1080	EXPECT_TOKEN(Tokens[17], tok::ampamp, TT_BinaryOperator);
1081	EXPECT_TOKEN(Tokens[19], tok::ampamp, TT_BinaryOperator);
1082
1083	Tokens = annotate(Code: "class Foo {\n"
1084	" void foo() {\n"
1085	" operator+(a * b);\n"
1086	" }\n"
1087	"}");
1088	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1089	EXPECT_TOKEN(Tokens[8], tok::kw_operator, TT_Unknown);
1090	EXPECT_TOKEN(Tokens[9], tok::plus, TT_OverloadedOperator);
1091	EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_OverloadedOperatorLParen);
1092	EXPECT_TOKEN(Tokens[12], tok::star, TT_BinaryOperator);
1093
1094	Tokens = annotate(Code: "std::vector<Foo> operator()(Foo &foo);");
1095	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1096	EXPECT_TOKEN(Tokens[3], tok::less, TT_TemplateOpener);
1097	EXPECT_TOKEN(Tokens[5], tok::greater, TT_TemplateCloser);
1098	EXPECT_TOKEN(Tokens[6], tok::kw_operator, TT_FunctionDeclarationName);
1099	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperator);
1100	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_OverloadedOperator);
1101	EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_OverloadedOperatorLParen);
1102	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
1103
1104	Tokens = annotate(Code: "decltype(auto) operator()(T &x);");
1105	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1106	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_TypeDeclarationParen);
1107	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_TypeDeclarationParen);
1108	EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);
1109	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperator);
1110	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_OverloadedOperator);
1111	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperatorLParen);
1112	EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
1113
1114	Tokens = annotate(Code: "friend ostream& ::operator<<(ostream& lhs, foo& rhs);");
1115	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1116	EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);
1117	EXPECT_TOKEN(Tokens[5], tok::lessless, TT_OverloadedOperator);
1118	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen);
1119	EXPECT_TOKEN(Tokens[8], tok::amp, TT_PointerOrReference);
1120	EXPECT_TOKEN(Tokens[12], tok::amp, TT_PointerOrReference);
1121
1122	Tokens = annotate(Code: "SomeLoooooooooooooooooType::Awaitable\n"
1123	"SomeLoooooooooooooooooType::operator co_await();");
1124	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1125	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_FunctionDeclarationName);
1126	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperatorLParen);
1127
1128	Tokens = annotate(Code: "using std::operator==;");
1129	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1130	// Not TT_FunctionDeclarationName.
1131	EXPECT_TOKEN(Tokens[3], tok::kw_operator, TT_Unknown);
1132
1133	Tokens = annotate(Code: "SomeAPI::operator()();");
1134	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1135	// Not TT_FunctionDeclarationName.
1136	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
1137	}
1138
1139	TEST_F(TokenAnnotatorTest, OverloadedOperatorInTemplate) {
1140	struct {
1141	const char *Text;
1142	tok::TokenKind Kind;
1143	} Operators[] = {{.Text: "+", .Kind: tok::plus},
1144	{.Text: "-", .Kind: tok::minus},
1145	// FIXME:
1146	// {"*", tok::star},
1147	{.Text: "/", .Kind: tok::slash},
1148	{.Text: "%", .Kind: tok::percent},
1149	{.Text: "^", .Kind: tok::caret},
1150	// FIXME:
1151	// {"&", tok::amp},
1152	{.Text: "|", .Kind: tok::pipe},
1153	{.Text: "~", .Kind: tok::tilde},
1154	{.Text: "!", .Kind: tok::exclaim},
1155	{.Text: "=", .Kind: tok::equal},
1156	// FIXME:
1157	// {"<", tok::less},
1158	{.Text: ">", .Kind: tok::greater},
1159	{.Text: "+=", .Kind: tok::plusequal},
1160	{.Text: "-=", .Kind: tok::minusequal},
1161	{.Text: "*=", .Kind: tok::starequal},
1162	{.Text: "/=", .Kind: tok::slashequal},
1163	{.Text: "%=", .Kind: tok::percentequal},
1164	{.Text: "^=", .Kind: tok::caretequal},
1165	{.Text: "&=", .Kind: tok::ampequal},
1166	{.Text: "|=", .Kind: tok::pipeequal},
1167	{.Text: "<<", .Kind: tok::lessless},
1168	{.Text: ">>", .Kind: tok::greatergreater},
1169	{.Text: ">>=", .Kind: tok::greatergreaterequal},
1170	{.Text: "<<=", .Kind: tok::lesslessequal},
1171	{.Text: "==", .Kind: tok::equalequal},
1172	{.Text: "!=", .Kind: tok::exclaimequal},
1173	{.Text: "<=", .Kind: tok::lessequal},
1174	{.Text: ">=", .Kind: tok::greaterequal},
1175	{.Text: "<=>", .Kind: tok::spaceship},
1176	{.Text: "&&", .Kind: tok::ampamp},
1177	{.Text: "||", .Kind: tok::pipepipe},
1178	{.Text: "++", .Kind: tok::plusplus},
1179	{.Text: "--", .Kind: tok::minusminus},
1180	{.Text: ",", .Kind: tok::comma},
1181	{.Text: "->*", .Kind: tok::arrowstar},
1182	{.Text: "->", .Kind: tok::arrow}};
1183
1184	for (const auto &Operator : Operators) {
1185	std::string Input("C<&operator");
1186	Input += Operator.Text;
1187	Input += " > a;";
1188	auto Tokens = annotate(Code: std::string(Input));
1189	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1190	EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
1191	EXPECT_TOKEN(Tokens[4], Operator.Kind, TT_OverloadedOperator);
1192	EXPECT_TOKEN(Tokens[5], tok::greater, TT_TemplateCloser);
1193	}
1194
1195	auto Tokens = annotate(Code: "C<&operator< <X>> lt;");
1196	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1197	EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
1198	EXPECT_TOKEN(Tokens[4], tok::less, TT_OverloadedOperator);
1199	EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);
1200	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
1201	EXPECT_TOKEN(Tokens[8], tok::greater, TT_TemplateCloser);
1202	}
1203
1204	TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
1205	auto Tokens = annotate(Code: "template <typename T>\n"
1206	"concept C = (Foo && Bar) && (Bar && Baz);");
1207
1208	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1209	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
1210	EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
1211	EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator);
1212
1213	Tokens = annotate(Code: "template <typename T>\n"
1214	"concept C = Foo && !Bar;");
1215
1216	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1217	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
1218	EXPECT_TOKEN(Tokens[10], tok::exclaim, TT_UnaryOperator);
1219
1220	Tokens = annotate(Code: "template <typename T>\n"
1221	"concept C = requires(T t) {\n"
1222	" { t.foo() };\n"
1223	"} && Bar<T> && Baz<T>;");
1224	ASSERT_EQ(Tokens.size(), 35u) << Tokens;
1225	EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
1226	EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
1227	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1228	EXPECT_TOKEN(Tokens[23], tok::ampamp, TT_BinaryOperator);
1229	EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);
1230
1231	Tokens = annotate(Code: "template<typename T>\n"
1232	"requires C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>\n"
1233	"struct Foo;");
1234	ASSERT_EQ(Tokens.size(), 36u) << Tokens;
1235	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1236	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
1237	EXPECT_EQ(Tokens[6]->FakeLParens.size(), 1u);
1238	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
1239	EXPECT_TOKEN(Tokens[16], tok::pipepipe, TT_BinaryOperator);
1240	EXPECT_TOKEN(Tokens[21], tok::ampamp, TT_BinaryOperator);
1241	EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator);
1242	// Not TT_TrailingAnnotation.
1243	EXPECT_TOKEN(Tokens[28], tok::identifier, TT_Unknown);
1244	EXPECT_TOKEN(Tokens[31], tok::greater, TT_TemplateCloser);
1245	EXPECT_EQ(Tokens[31]->FakeRParens, 1u);
1246	EXPECT_TRUE(Tokens[31]->ClosesRequiresClause);
1247	// Not TT_TrailingAnnotation.
1248	EXPECT_TOKEN(Tokens[33], tok::identifier, TT_Unknown);
1249
1250	Tokens =
1251	annotate(Code: "template <typename T>\n"
1252	" requires(C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>)\n"
1253	"struct Foo;");
1254	ASSERT_EQ(Tokens.size(), 38u) << Tokens;
1255	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1256	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_Unknown);
1257	EXPECT_EQ(Tokens[7]->FakeLParens.size(), 1u);
1258	EXPECT_TOKEN(Tokens[11], tok::ampamp, TT_BinaryOperator);
1259	EXPECT_TOKEN(Tokens[17], tok::pipepipe, TT_BinaryOperator);
1260	EXPECT_TOKEN(Tokens[22], tok::ampamp, TT_BinaryOperator);
1261	EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);
1262	EXPECT_TOKEN(Tokens[32], tok::greater, TT_TemplateCloser);
1263	EXPECT_EQ(Tokens[32]->FakeRParens, 1u);
1264	EXPECT_TOKEN(Tokens[33], tok::r_paren, TT_Unknown);
1265	EXPECT_TRUE(Tokens[33]->ClosesRequiresClause);
1266	EXPECT_TOKEN(Tokens[35], tok::identifier, TT_Unknown);
1267
1268	Tokens = annotate(Code: "template <typename T>\n"
1269	"void foo(T) noexcept requires Bar<T>;");
1270	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1271	EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause);
1272
1273	Tokens = annotate(Code: "template <typename T>\n"
1274	"requires Bar<T> || Baz<T>\n"
1275	"auto foo(T) -> int;");
1276	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
1277	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1278	EXPECT_EQ(Tokens[11]->FakeLParens.size(), 0u);
1279	EXPECT_TRUE(Tokens[14]->ClosesRequiresClause);
1280	EXPECT_TOKEN(Tokens[20], tok::arrow, TT_TrailingReturnArrow);
1281
1282	Tokens = annotate(Code: "template <typename T>\n"
1283	"requires Bar<T>\n"
1284	"bool foo(T) { return false; }");
1285	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1286	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1287	EXPECT_TRUE(Tokens[9]->ClosesRequiresClause);
1288	EXPECT_TOKEN(Tokens[11], tok::identifier, TT_FunctionDeclarationName);
1289	EXPECT_TOKEN(Tokens[12], tok::l_paren, TT_FunctionDeclarationLParen);
1290
1291	Tokens = annotate(Code: "template <typename T>\n"
1292	"requires Bar<T>\n"
1293	"decltype(auto) foo(T) { return false; }");
1294	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
1295	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1296	EXPECT_TRUE(Tokens[9]->ClosesRequiresClause);
1297	EXPECT_TOKEN(Tokens[14], tok::identifier, TT_FunctionDeclarationName);
1298	EXPECT_TOKEN(Tokens[15], tok::l_paren, TT_FunctionDeclarationLParen);
1299
1300	Tokens = annotate(Code: "template <typename T>\n"
1301	"struct S {\n"
1302	" void foo() const requires Bar<T>;\n"
1303	" void bar() const & requires Baz<T>;\n"
1304	" void bar() && requires Baz2<T>;\n"
1305	" void baz() const & noexcept requires Baz<T>;\n"
1306	" void baz() && noexcept requires Baz2<T>;\n"
1307	"};\n"
1308	"\n"
1309	"void S::bar() const & requires Baz<T> { }");
1310	ASSERT_EQ(Tokens.size(), 85u) << Tokens;
1311	EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
1312	EXPECT_TOKEN(Tokens[24], tok::amp, TT_PointerOrReference);
1313	EXPECT_TOKEN(Tokens[25], tok::kw_requires, TT_RequiresClause);
1314	EXPECT_TOKEN(Tokens[35], tok::ampamp, TT_PointerOrReference);
1315	EXPECT_TOKEN(Tokens[36], tok::kw_requires, TT_RequiresClause);
1316	EXPECT_TOKEN(Tokens[47], tok::amp, TT_PointerOrReference);
1317	EXPECT_TOKEN(Tokens[49], tok::kw_requires, TT_RequiresClause);
1318	EXPECT_TOKEN(Tokens[59], tok::ampamp, TT_PointerOrReference);
1319	EXPECT_TOKEN(Tokens[61], tok::kw_requires, TT_RequiresClause);
1320	EXPECT_TOKEN(Tokens[76], tok::amp, TT_PointerOrReference);
1321	EXPECT_TOKEN(Tokens[77], tok::kw_requires, TT_RequiresClause);
1322
1323	Tokens = annotate(Code: "void Class::member() && requires(Constant) {}");
1324	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1325	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1326
1327	Tokens = annotate(Code: "void Class::member() && requires(Constant<T>) {}");
1328	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1329	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1330
1331	Tokens =
1332	annotate(Code: "void Class::member() && requires(Namespace::Constant<T>) {}");
1333	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1334	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1335
1336	Tokens = annotate(Code: "void Class::member() && requires(typename "
1337	"Namespace::Outer<T>::Inner::Constant) {}");
1338	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
1339	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1340
1341	Tokens = annotate(Code: "struct [[nodiscard]] zero_t {\n"
1342	" template<class T>\n"
1343	" requires requires { number_zero_v<T>; }\n"
1344	" [[nodiscard]] constexpr operator T() const { "
1345	"return number_zero_v<T>; }\n"
1346	"};");
1347	ASSERT_EQ(Tokens.size(), 44u) << Tokens;
1348	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);
1349	EXPECT_TOKEN(Tokens[11], tok::identifier, TT_Unknown);
1350	EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
1351	EXPECT_TOKEN(Tokens[14], tok::kw_requires, TT_RequiresExpression);
1352	EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_RequiresExpressionLBrace);
1353	EXPECT_TOKEN(Tokens[21], tok::r_brace, TT_Unknown);
1354	EXPECT_EQ(Tokens[21]->MatchingParen, Tokens[15]);
1355	EXPECT_TRUE(Tokens[21]->ClosesRequiresClause);
1356
1357	Tokens = annotate(Code: "template <typename Foo>\n"
1358	"void Fun(const Foo &F)\n"
1359	" requires requires(Foo F) {\n"
1360	" { F.Bar() } -> std::same_as<int>;\n"
1361	" };");
1362	ASSERT_EQ(Tokens.size(), 38u) << Tokens;
1363	EXPECT_TOKEN(Tokens[19], tok::l_brace, TT_RequiresExpressionLBrace);
1364
1365	Tokens =
1366	annotate(Code: "template <class A, class B> concept C ="
1367	"std::same_as<std::iter_value_t<A>, std::iter_value_t<B>>;");
1368	ASSERT_EQ(Tokens.size(), 31u) << Tokens;
1369	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
1370	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
1371	EXPECT_TOKEN(Tokens[8], tok::kw_concept, TT_Unknown);
1372	EXPECT_TOKEN(Tokens[14], tok::less, TT_TemplateOpener);
1373	EXPECT_TOKEN(Tokens[18], tok::less, TT_TemplateOpener);
1374	EXPECT_TOKEN(Tokens[20], tok::greater, TT_TemplateCloser);
1375	EXPECT_TOKEN(Tokens[25], tok::less, TT_TemplateOpener);
1376	EXPECT_TOKEN(Tokens[27], tok::greater, TT_TemplateCloser);
1377	EXPECT_TOKEN(Tokens[28], tok::greater, TT_TemplateCloser);
1378
1379	Tokens = annotate(Code: "auto bar() -> int requires(is_integral_v<T>) {}");
1380	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1381	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1382
1383	Tokens = annotate(Code: "auto bar() -> void requires(is_integral_v<T>) {}");
1384	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1385	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1386
1387	Tokens = annotate(Code: "auto bar() -> MyType requires(is_integral_v<T>) {}");
1388	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1389	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1390
1391	Tokens =
1392	annotate(Code: "auto bar() -> SOME_MACRO_TYPE requires(is_integral_v<T>) {}");
1393	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1394	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1395
1396	Tokens =
1397	annotate(Code: "auto bar() -> qualified::type requires(is_integral_v<T>) {}");
1398	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1399	EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresClause);
1400
1401	Tokens =
1402	annotate(Code: "auto bar() -> Template<type> requires(is_integral_v<T>) {}");
1403	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1404	EXPECT_TOKEN(Tokens[9], tok::kw_requires, TT_RequiresClause);
1405
1406	Tokens = annotate(Code: "void foo() requires((A<T>) && C) {}");
1407	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1408	EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1409	EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_BinaryOperator);
1410
1411	Tokens = annotate(Code: "void foo() requires(((A<T>) && C)) {}");
1412	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1413	EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1414	EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
1415
1416	Tokens = annotate(Code: "void foo() requires([](T&&){}(t)) {}");
1417	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1418	EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1419	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
1420
1421	Tokens = annotate(Code: "void foo() requires([](T&& u){}(t)) {}");
1422	ASSERT_EQ(Tokens.size(), 22u) << Tokens;
1423	EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1424	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
1425
1426	Tokens = annotate(Code: "void f() & requires(true) {}");
1427	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1428	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
1429	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1430
1431	Tokens = annotate(Code: "void f() & requires(C<true, true>) {}");
1432	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1433	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
1434	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1435
1436	Tokens = annotate(Code: "template <typename T>\n"
1437	"concept C = (!Foo<T>) && Bar;");
1438	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1439	EXPECT_TOKEN(Tokens[15], tok::ampamp, TT_BinaryOperator);
1440
1441	Tokens = annotate(Code: "void f() & requires(C<decltype(x)>) {}");
1442	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1443	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
1444	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1445
1446	Tokens = annotate(Code: "auto f() -> int& requires(C<decltype(x)>) {}");
1447	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1448	EXPECT_TOKEN(Tokens[6], tok::amp, TT_PointerOrReference);
1449	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1450
1451	Tokens = annotate(Code: "bool x = t && requires(decltype(t) x) { x.foo(); };");
1452	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
1453	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresExpression);
1454
1455	Tokens = annotate(Code: "bool x = t && requires(Foo<decltype(t)> x) { x.foo(); };");
1456	ASSERT_EQ(Tokens.size(), 26u) << Tokens;
1457	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresExpression);
1458
1459	Tokens = annotate(Code: "bool x = t && requires(Foo<C1 || C2> x) { x.foo(); };");
1460	ASSERT_EQ(Tokens.size(), 25u) << Tokens;
1461	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresExpression);
1462
1463	// Second function definition is required due to lookahead
1464	Tokens = annotate(Code: "void f() &\n"
1465	" requires(n == 1)\n"
1466	"{}\n"
1467	"void g();");
1468	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1469	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
1470	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1471
1472	Tokens = annotate(Code: "void foo() &&\n"
1473	" requires(!bar)\n"
1474	"{\n"
1475	" baz();\n"
1476	"}");
1477	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1478	EXPECT_TOKEN(Tokens[4], tok::ampamp, TT_PointerOrReference);
1479	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1480
1481	Tokens = annotate(Code: "auto foo() -> auto *\n"
1482	" requires(not bar)\n"
1483	"{\n"
1484	" return baz;\n"
1485	"}");
1486	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1487	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1488	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);
1489	}
1490
1491	TEST_F(TokenAnnotatorTest, UnderstandsRequiresExpressions) {
1492	auto Tokens = annotate(Code: "bool b = requires(int i) { i + 5; };");
1493	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1494	EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1495	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1496	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1497
1498	Tokens = annotate(Code: "if (requires(int i) { i + 5; }) return;");
1499	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1500	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1501	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1502	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_RequiresExpressionLBrace);
1503
1504	Tokens = annotate(Code: "if (func() && requires(int i) { i + 5; }) return;");
1505	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1506	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresExpression);
1507	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_RequiresExpressionLParen);
1508	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_RequiresExpressionLBrace);
1509
1510	Tokens = annotate(Code: "foo(requires(const T t) {});");
1511	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1512	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1513	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1514	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1515
1516	Tokens = annotate(Code: "foo(requires(const int t) {});");
1517	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1518	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1519	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1520	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1521
1522	Tokens = annotate(Code: "foo(requires(const T t) {});");
1523	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1524	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1525	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1526	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1527
1528	Tokens = annotate(Code: "foo(requires(int const* volatile t) {});");
1529	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1530	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1531	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1532	EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);
1533	EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
1534
1535	Tokens = annotate(Code: "foo(requires(T const* volatile t) {});");
1536	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1537	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1538	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1539	EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);
1540	EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
1541
1542	Tokens = annotate(Code: "foo(requires(T& t) {});");
1543	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1544	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1545	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1546	EXPECT_TOKEN(Tokens[5], tok::amp, TT_PointerOrReference);
1547	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1548
1549	Tokens = annotate(Code: "foo(requires(T&& t) {});");
1550	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1551	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1552	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1553	EXPECT_TOKEN(Tokens[5], tok::ampamp, TT_PointerOrReference);
1554	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1555
1556	Tokens = annotate(Code: "bool foo = requires(T& t) {};");
1557	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1558	EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1559	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1560	EXPECT_TOKEN(Tokens[6], tok::amp, TT_PointerOrReference);
1561	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_RequiresExpressionLBrace);
1562
1563	Tokens = annotate(Code: "bool foo = requires(T&& t) {};");
1564	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1565	EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1566	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1567	EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference);
1568	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_RequiresExpressionLBrace);
1569
1570	Tokens =
1571	annotate(Code: "foo(requires(const typename Outer<T>::Inner * const t) {});");
1572	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1573	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1574	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1575	EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
1576	EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_RequiresExpressionLBrace);
1577
1578	Tokens = annotate(Code: "template <typename T>\n"
1579	"concept C = requires(T T) {\n"
1580	" requires Bar<T> && Foo<T>;\n"
1581	"};");
1582	ASSERT_EQ(Tokens.size(), 28u) << Tokens;
1583	EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
1584	EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
1585	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1586	EXPECT_TOKEN(Tokens[14], tok::kw_requires,
1587	TT_RequiresClauseInARequiresExpression);
1588
1589	Tokens = annotate(Code: "template <typename T>\n"
1590	"concept C = requires(T T) {\n"
1591	" { t.func() } -> std::same_as<int>;"
1592	" requires Bar<T> && Foo<T>;\n"
1593	"};");
1594	ASSERT_EQ(Tokens.size(), 43u) << Tokens;
1595	EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
1596	EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
1597	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1598	EXPECT_TOKEN(Tokens[29], tok::kw_requires,
1599	TT_RequiresClauseInARequiresExpression);
1600
1601	// Invalid Code, but we don't want to crash. See http://llvm.org/PR54350.
1602	Tokens = annotate(Code: "bool r10 = requires (struct new_struct { int x; } s) { "
1603	"requires true; };");
1604	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1605	EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1606	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1607	EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_RequiresExpressionLBrace);
1608
1609	Tokens = annotate(Code: "bool foo = requires(C<true, true> c) {\n"
1610	" { c.foo(); }\n"
1611	"};");
1612	ASSERT_EQ(Tokens.size(), 25u) << Tokens;
1613	EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1614	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1615	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1616	}
1617
1618	TEST_F(TokenAnnotatorTest, UnderstandsPragmaRegion) {
1619	// Everything after #pragma region should be ImplicitStringLiteral
1620	auto Tokens = annotate(Code: "#pragma region Foo(Bar: Hello)");
1621	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1622	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_ImplicitStringLiteral);
1623	EXPECT_TOKEN(Tokens[6], tok::colon, TT_ImplicitStringLiteral);
1624	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_ImplicitStringLiteral);
1625
1626	// Make sure it's annotated correctly inside a function as well
1627	Tokens = annotate(Code: "void test(){\n#pragma region Foo(Bar: Hello)\n}");
1628	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1629	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_ImplicitStringLiteral);
1630	EXPECT_TOKEN(Tokens[11], tok::colon, TT_ImplicitStringLiteral);
1631	EXPECT_TOKEN(Tokens[12], tok::identifier, TT_ImplicitStringLiteral);
1632	}
1633
1634	TEST_F(TokenAnnotatorTest, RequiresDoesNotChangeParsingOfTheRest) {
1635	const char *BaseCode = nullptr;
1636	const char *ConstrainedCode = nullptr;
1637	auto BaseTokenCount = 0u;
1638	auto RequiresTokenCount = 0u;
1639	auto PrefixTokenCount = 0u;
1640
1641	auto TestRequires = [&](int Line) {
1642	const auto BaseTokens = annotate(Code: BaseCode);
1643	const auto ConstrainedTokens = annotate(Code: ConstrainedCode);
1644
1645	#define LINE " (Line " << Line << ')'
1646
1647	ASSERT_EQ(BaseTokens.size(), BaseTokenCount) << BaseTokens << LINE;
1648	ASSERT_EQ(ConstrainedTokens.size(), BaseTokenCount + RequiresTokenCount)
1649	<< LINE;
1650
1651	for (auto I = 0u; I < BaseTokenCount; ++I) {
1652	EXPECT_EQ(
1653	*BaseTokens[I],
1654	*ConstrainedTokens[I < PrefixTokenCount ? I : I + RequiresTokenCount])
1655	<< I << LINE;
1656	}
1657
1658	#undef LINE
1659	};
1660
1661	BaseCode = "template<typename T>\n"
1662	"T Pi = 3.14;";
1663	ConstrainedCode = "template<typename T>\n"
1664	" requires Foo<T>\n"
1665	"T Pi = 3.14;";
1666	BaseTokenCount = 11;
1667	RequiresTokenCount = 5;
1668	PrefixTokenCount = 5;
1669	TestRequires(__LINE__);
1670
1671	BaseCode = "template<typename T>\n"
1672	"struct Bar;";
1673	ConstrainedCode = "template<typename T>\n"
1674	" requires Foo<T>\n"
1675	"struct Bar;";
1676	BaseTokenCount = 9;
1677	TestRequires(__LINE__);
1678
1679	BaseCode = "template<typename T>\n"
1680	"struct Bar {\n"
1681	" T foo();\n"
1682	" T bar();\n"
1683	"};";
1684	ConstrainedCode = "template<typename T>\n"
1685	" requires Foo<T>\n"
1686	"struct Bar {\n"
1687	" T foo();\n"
1688	" T bar();\n"
1689	"};";
1690	BaseTokenCount = 21;
1691	TestRequires(__LINE__);
1692
1693	BaseCode = "template<typename T>\n"
1694	"Bar(T) -> Bar<T>;";
1695	ConstrainedCode = "template<typename T>\n"
1696	" requires Foo<T>\n"
1697	"Bar(T) -> Bar<T>;";
1698	BaseTokenCount = 16;
1699	TestRequires(__LINE__);
1700
1701	BaseCode = "template<typename T>\n"
1702	"T foo();";
1703	ConstrainedCode = "template<typename T>\n"
1704	" requires Foo<T>\n"
1705	"T foo();";
1706	BaseTokenCount = 11;
1707	TestRequires(__LINE__);
1708
1709	BaseCode = "template<typename T>\n"
1710	"T foo() {\n"
1711	" auto bar = baz();\n"
1712	" return bar + T{};\n"
1713	"}";
1714	ConstrainedCode = "template<typename T>\n"
1715	" requires Foo<T>\n"
1716	"T foo() {\n"
1717	" auto bar = baz();\n"
1718	" return bar + T{};\n"
1719	"}";
1720	BaseTokenCount = 26;
1721	TestRequires(__LINE__);
1722
1723	BaseCode = "template<typename T>\n"
1724	"T foo();";
1725	ConstrainedCode = "template<typename T>\n"
1726	"T foo() requires Foo<T>;";
1727	BaseTokenCount = 11;
1728	PrefixTokenCount = 9;
1729	TestRequires(__LINE__);
1730
1731	BaseCode = "template<typename T>\n"
1732	"T foo() {\n"
1733	" auto bar = baz();\n"
1734	" return bar + T{};\n"
1735	"}";
1736	ConstrainedCode = "template<typename T>\n"
1737	"T foo() requires Foo<T> {\n"
1738	" auto bar = baz();\n"
1739	" return bar + T{};\n"
1740	"}";
1741	BaseTokenCount = 26;
1742	TestRequires(__LINE__);
1743
1744	BaseCode = "template<typename T>\n"
1745	"T foo();";
1746	ConstrainedCode = "template<typename T>\n"
1747	" requires(Foo<T>)\n"
1748	"T foo();";
1749	BaseTokenCount = 11;
1750	RequiresTokenCount = 7;
1751	PrefixTokenCount = 5;
1752	TestRequires(__LINE__);
1753
1754	BaseCode = "template<typename T>\n"
1755	"Bar(T) -> Bar<typename T::I>;";
1756	ConstrainedCode = "template<typename T>\n"
1757	" requires requires(T &&t) {\n"
1758	" typename T::I;\n"
1759	" }\n"
1760	"Bar(T) -> Bar<typename T::I>;";
1761	BaseTokenCount = 19;
1762	RequiresTokenCount = 14;
1763	PrefixTokenCount = 5;
1764	TestRequires(__LINE__);
1765
1766	BaseCode = "struct [[nodiscard]] zero_t {\n"
1767	" template<class T>\n"
1768	" [[nodiscard]] constexpr operator T() const { return v<T>; }\n"
1769	"};";
1770	ConstrainedCode =
1771	"struct [[nodiscard]] zero_t {\n"
1772	" template<class T>\n"
1773	" requires requires { v<T>; }\n"
1774	" [[nodiscard]] constexpr operator T() const { return v<T>; }\n"
1775	"};";
1776	BaseTokenCount = 35;
1777	RequiresTokenCount = 9;
1778	PrefixTokenCount = 13;
1779	TestRequires(__LINE__);
1780
1781	BaseCode = "constexpr Foo(Foo const &other)\n"
1782	" : value{other.value} {\n"
1783	" do_magic();\n"
1784	" do_more_magic();\n"
1785	"}";
1786	ConstrainedCode = "constexpr Foo(Foo const &other)\n"
1787	" requires std::is_copy_constructible<T>\n"
1788	" : value{other.value} {\n"
1789	" do_magic();\n"
1790	" do_more_magic();\n"
1791	"}";
1792	BaseTokenCount = 26;
1793	RequiresTokenCount = 7;
1794	PrefixTokenCount = 8;
1795	TestRequires(__LINE__);
1796
1797	BaseCode = "constexpr Foo(Foo const &other)\n"
1798	" : value{other.value} {\n"
1799	" do_magic();\n"
1800	" do_more_magic();\n"
1801	"}";
1802	ConstrainedCode = "constexpr Foo(Foo const &other)\n"
1803	" requires (std::is_copy_constructible<T>)\n"
1804	" : value{other.value} {\n"
1805	" do_magic();\n"
1806	" do_more_magic();\n"
1807	"}";
1808	RequiresTokenCount = 9;
1809	TestRequires(__LINE__);
1810
1811	BaseCode = "template<typename T>\n"
1812	"ANNOTATE(\"S\"\n"
1813	" \"S\")\n"
1814	"void foo();";
1815	ConstrainedCode = "template<typename T>\n"
1816	" requires(true)\n"
1817	"ANNOTATE(\"S\"\n"
1818	" \"S\")\n"
1819	"void foo();";
1820	BaseTokenCount = 16;
1821	RequiresTokenCount = 4;
1822	PrefixTokenCount = 5;
1823	TestRequires(__LINE__);
1824	}
1825
1826	TEST_F(TokenAnnotatorTest, UnderstandsAsm) {
1827	auto Tokens = annotate(Code: "__asm{\n"
1828	"\"a\":\n"
1829	": x\n"
1830	":};");
1831	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1832	EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1833	EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_InlineASMBrace);
1834	EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
1835	EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
1836	EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
1837	EXPECT_TOKEN(Tokens[7], tok::r_brace, TT_InlineASMBrace);
1838
1839	Tokens = annotate(Code: "__asm(\n"
1840	"\"a\":\n"
1841	": x\n"
1842	":);");
1843	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1844	EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1845	EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
1846	EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
1847	EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
1848
1849	Tokens = annotate(Code: "asm volatile (\n"
1850	"\"a_label:\"\n"
1851	":\n"
1852	": x\n"
1853	":);");
1854	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1855	EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1856	EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
1857	EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
1858	EXPECT_TOKEN(Tokens[7], tok::colon, TT_InlineASMColon);
1859
1860	Tokens = annotate(Code: "__asm__(\n"
1861	"\"a_label:\"\n"
1862	": x\n"
1863	":\n"
1864	": y);");
1865	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1866	EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1867	EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
1868	EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
1869	EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
1870
1871	Tokens = annotate(Code: "__asm volatile (\n"
1872	"\"a_label:\"\n"
1873	"\"a b c(%%x)\"\n"
1874	":\n"
1875	": x\n"
1876	":);");
1877	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1878	EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1879	EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
1880	EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
1881	EXPECT_TOKEN(Tokens[8], tok::colon, TT_InlineASMColon);
1882
1883	Tokens = annotate(Code: "asm(\n"
1884	"\"insn\"\n"
1885	": \"=r\" (var1), \"=&r\" (value)\n"
1886	":\n"
1887	": \"memory\");");
1888	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1889	EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1890	EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
1891	EXPECT_TOKEN(Tokens[13], tok::colon, TT_InlineASMColon);
1892	EXPECT_TOKEN(Tokens[14], tok::colon, TT_InlineASMColon);
1893
1894	Tokens = annotate(Code: "__asm__ volatile (\n"
1895	"\"ldr r1, [r0, %%[sym]]\"\n"
1896	":\n"
1897	": [sym] \"J\" (aaaaa(aaaa, aaaa))\n"
1898	");");
1899	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1900	EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1901	EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
1902	EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
1903	EXPECT_TOKEN(Tokens[6], tok::l_square, TT_InlineASMSymbolicNameLSquare);
1904	}
1905
1906	TEST_F(TokenAnnotatorTest, UnderstandsObjCBlock) {
1907	auto Tokens = annotate(Code: "int (^)() = ^ ()\n"
1908	" external_source_symbol() { //\n"
1909	" return 1;\n"
1910	"};");
1911	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1912	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ObjCBlockLParen);
1913	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_ObjCBlockLBrace);
1914
1915	Tokens = annotate(Code: "int *p = ^int*(){ //\n"
1916	" return nullptr;\n"
1917	"}();");
1918	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1919	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_ObjCBlockLBrace);
1920
1921	Tokens = annotate(Code: "id (^block)(Foo *a) = ^id _Nullable(Foo *_Nullable a) {\n"
1922	" return a;\n"
1923	"};");
1924	ASSERT_EQ(Tokens.size(), 27u) << Tokens;
1925	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown); // Not CtorDtorDeclName.
1926	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ObjCBlockLParen);
1927	}
1928
1929	TEST_F(TokenAnnotatorTest, UnderstandsObjCMethodExpr) {
1930	auto Tokens = annotate(Code: "void f() {\n"
1931	" //\n"
1932	" BOOL a = [b.c n] > 1;\n"
1933	"}");
1934	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1935	EXPECT_TOKEN(Tokens[9], tok::l_square, TT_ObjCMethodExpr);
1936	EXPECT_TOKEN(Tokens[15], tok::greater, TT_BinaryOperator);
1937	}
1938
1939	TEST_F(TokenAnnotatorTest, UnderstandsObjCMethodDecl) {
1940	auto Tokens = annotate(Code: "/**/ - (void)foo;");
1941	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1942	EXPECT_TOKEN(Tokens[1], tok::minus, TT_ObjCMethodSpecifier);
1943	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_SelectorName);
1944	}
1945
1946	TEST_F(TokenAnnotatorTest, UnderstandsLambdas) {
1947	auto Tokens = annotate(Code: "[]() constexpr {}");
1948	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1949	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1950	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1951	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1952
1953	Tokens = annotate(Code: "[]() consteval {}");
1954	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1955	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1956	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1957	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1958
1959	Tokens = annotate(Code: "[]() mutable {}");
1960	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1961	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1962	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1963	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1964
1965	Tokens = annotate(Code: "[]() static {}");
1966	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1967	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1968	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1969	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1970
1971	Tokens = annotate(Code: "[]() -> auto {}");
1972	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1973	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1974	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1975	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
1976	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);
1977
1978	Tokens = annotate(Code: "[]() -> auto & {}");
1979	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1980	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1981	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1982	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
1983	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1984
1985	Tokens = annotate(Code: "[]() -> auto * {}");
1986	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1987	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1988	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1989	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
1990	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1991
1992	Tokens = annotate(Code: "[] {}");
1993	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
1994	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1995	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_LambdaLBrace);
1996
1997	Tokens = annotate(Code: "[] noexcept {}");
1998	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
1999	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2000	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_LambdaLBrace);
2001
2002	Tokens = annotate(Code: "[] -> auto {}");
2003	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2004	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2005	EXPECT_TOKEN(Tokens[2], tok::arrow, TT_LambdaArrow);
2006	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_LambdaLBrace);
2007
2008	Tokens = annotate(Code: "[] -> struct S { return {}; }");
2009	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2010	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2011	EXPECT_TOKEN(Tokens[2], tok::arrow, TT_LambdaArrow);
2012	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);
2013	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
2014
2015	Tokens = annotate(Code: "foo([&](u32 bar) __attribute__((attr)) -> void {});");
2016	ASSERT_EQ(Tokens.size(), 22u) << Tokens;
2017	EXPECT_TOKEN(Tokens[2], tok::l_square, TT_LambdaLSquare);
2018	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_LambdaDefinitionLParen);
2019	EXPECT_TOKEN(Tokens[15], tok::arrow, TT_LambdaArrow);
2020	EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);
2021
2022	Tokens = annotate(Code: "[] <typename T> () {}");
2023	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2024	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2025	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2026	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);
2027	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_LambdaLBrace);
2028
2029	Tokens = annotate(Code: "[] <typename T> {}");
2030	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2031	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2032	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2033	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);
2034
2035	Tokens = annotate(Code: "[] <typename... T> () {}");
2036	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2037	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2038	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2039	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_LambdaDefinitionLParen);
2040	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
2041
2042	Tokens = annotate(Code: "[] <typename... T> {}");
2043	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2044	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2045	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2046	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
2047
2048	Tokens = annotate(Code: "[] <int... T> () {}");
2049	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2050	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2051	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2052	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_LambdaDefinitionLParen);
2053	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
2054
2055	Tokens = annotate(Code: "[] <int... T> {}");
2056	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2057	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2058	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2059	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
2060
2061	Tokens = annotate(Code: "[] <Foo... T> () {}");
2062	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2063	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2064	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2065	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_LambdaDefinitionLParen);
2066	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
2067
2068	Tokens = annotate(Code: "[] <Foo... T> {}");
2069	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2070	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2071	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2072	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
2073
2074	// Lambdas with a requires-clause
2075	Tokens = annotate(Code: "[] <typename T> (T t) requires Bar<T> {}");
2076	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
2077	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2078	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2079	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);
2080	EXPECT_TOKEN(Tokens[10], tok::kw_requires, TT_RequiresClause);
2081	EXPECT_TRUE(Tokens[14]->ClosesRequiresClause);
2082	EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
2083
2084	Tokens = annotate(Code: "[] <typename T> (T &&t) requires Bar<T> {}");
2085	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
2086	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2087	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2088	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);
2089	EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_PointerOrReference);
2090	EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause);
2091	EXPECT_TRUE(Tokens[15]->ClosesRequiresClause);
2092	EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_LambdaLBrace);
2093
2094	Tokens = annotate(Code: "[] <typename T> (T t) requires Foo<T> || Bar<T> {}");
2095	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
2096	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2097	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2098	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);
2099	EXPECT_TOKEN(Tokens[10], tok::kw_requires, TT_RequiresClause);
2100	EXPECT_TRUE(Tokens[19]->ClosesRequiresClause);
2101	EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
2102
2103	Tokens = annotate(Code: "[] <typename T> (T t) -> T requires Bar<T> {}");
2104	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
2105	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2106	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2107	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);
2108	EXPECT_TOKEN(Tokens[10], tok::arrow, TT_LambdaArrow);
2109	EXPECT_TOKEN(Tokens[12], tok::kw_requires, TT_RequiresClause);
2110	EXPECT_TRUE(Tokens[16]->ClosesRequiresClause);
2111	EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);
2112
2113	Tokens = annotate(Code: "[] <typename T> requires Bar<T> (T t) {}");
2114	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
2115	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2116	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2117	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2118	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
2119	// FIXME:
2120	// EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_LambdaDefinitionLParen);
2121	EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
2122
2123	Tokens = annotate(Code: "[] <typename T> requires Bar<T> (T &&t) {}");
2124	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
2125	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2126	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2127	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2128	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
2129	// FIXME:
2130	// EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_LambdaDefinitionLParen);
2131	EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_PointerOrReference);
2132	EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_LambdaLBrace);
2133
2134	Tokens = annotate(Code: "[] <typename T> requires Foo<T> || Bar<T> (T t) {}");
2135	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
2136	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2137	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2138	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2139	EXPECT_TRUE(Tokens[15]->ClosesRequiresClause);
2140	// FIXME:
2141	// EXPECT_TOKEN(Tokens[16], tok::l_paren, TT_LambdaDefinitionLParen);
2142	EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
2143
2144	Tokens = annotate(Code: "[] <typename T> requires true (T&& t) {}");
2145	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2146	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2147	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2148	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2149	EXPECT_TRUE(Tokens[7]->ClosesRequiresClause);
2150	// FIXME:
2151	// EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);
2152	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
2153	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_LambdaLBrace);
2154
2155	Tokens = annotate(Code: "[] <typename T> requires Bar<T> {}");
2156	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2157	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2158	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2159	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2160	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
2161	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_LambdaLBrace);
2162
2163	Tokens = annotate(Code: "[] <typename T> requires Bar<T> noexcept {}");
2164	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2165	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2166	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2167	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2168	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
2169	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
2170
2171	Tokens = annotate(Code: "[] <typename T> requires Bar<T> -> T {}");
2172	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2173	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2174	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2175	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2176	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
2177	EXPECT_TOKEN(Tokens[11], tok::arrow, TT_LambdaArrow);
2178	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_LambdaLBrace);
2179
2180	Tokens = annotate(Code: "[] <typename T> requires Foo<T> (T t) requires Bar<T> {}");
2181	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
2182	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2183	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2184	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2185	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
2186	// FIXME:
2187	// EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_LambdaDefinitionLParen);
2188	EXPECT_TOKEN(Tokens[15], tok::kw_requires, TT_RequiresClause);
2189	EXPECT_TRUE(Tokens[19]->ClosesRequiresClause);
2190	EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
2191
2192	Tokens = annotate(Code: "[] <typename T = int> (T t) {}");
2193	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2194	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2195	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2196	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
2197	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);
2198	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
2199
2200	Tokens = annotate(Code: "[] <int I = 0> (T t) {}");
2201	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2202	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2203	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2204	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
2205	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);
2206	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
2207
2208	Tokens = annotate(Code: "[] <bool b = false> (T t) {}");
2209	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2210	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2211	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2212	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
2213	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);
2214	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
2215
2216	Tokens = annotate(Code: "[] <bool b = true && false> (T&& t) {}");
2217	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
2218	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2219	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2220	EXPECT_TOKEN(Tokens[7], tok::ampamp, TT_BinaryOperator);
2221	EXPECT_TOKEN(Tokens[9], tok::greater, TT_TemplateCloser);
2222	EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_LambdaDefinitionLParen);
2223	EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_PointerOrReference);
2224	EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
2225
2226	Tokens = annotate(Code: "[] <typename T = int> requires Foo<T> (T t) {}");
2227	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
2228	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2229	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2230	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
2231	EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresClause);
2232	// FIXME:
2233	// EXPECT_TOKEN(Tokens[13], tok::l_paren, TT_LambdaDefinitionLParen);
2234	EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);
2235
2236	Tokens = annotate(Code: "auto foo{(std::function<int()>)[] { return 0; }};");
2237	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
2238	EXPECT_TOKEN(Tokens[13], tok::l_square, TT_LambdaLSquare);
2239	EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
2240
2241	Tokens = annotate(Code: "auto foo{(int (*)())[] { return 0; }};");
2242	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
2243	EXPECT_TOKEN(Tokens[11], tok::l_square, TT_LambdaLSquare);
2244	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_LambdaLBrace);
2245
2246	Tokens = annotate(Code: "SomeFunction({[]() -> int *[] { return {}; }});");
2247	ASSERT_EQ(Tokens.size(), 22u) << Tokens;
2248	EXPECT_TOKEN(Tokens[3], tok::l_square, TT_LambdaLSquare);
2249	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_LambdaDefinitionLParen);
2250	EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);
2251
2252	Tokens = annotate(Code: "foo = bar * [] { return 2; }();");
2253	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2254	EXPECT_TOKEN(Tokens[3], tok::star, TT_BinaryOperator);
2255	EXPECT_TOKEN(Tokens[4], tok::l_square, TT_LambdaLSquare);
2256	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);
2257	}
2258
2259	TEST_F(TokenAnnotatorTest, UnderstandsFunctionAnnotations) {
2260	auto Tokens = annotate(Code: "template <typename T>\n"
2261	"DEPRECATED(\"Use NewClass::NewFunction instead.\")\n"
2262	"string OldFunction(const string &parameter) {}");
2263	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
2264	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_FunctionAnnotationRParen);
2265
2266	Tokens = annotate(Code: "template <typename T>\n"
2267	"A(T) noexcept;");
2268	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2269	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
2270
2271	Tokens = annotate(Code: "FOO(bar)();");
2272	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2273	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_Unknown);
2274	}
2275
2276	TEST_F(TokenAnnotatorTest, UnderstandsFunctionDeclarationNames) {
2277	auto Tokens = annotate(Code: "void f [[noreturn]] ();");
2278	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2279	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2280	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);
2281
2282	Tokens = annotate(Code: "void f [[noreturn]] () {}");
2283	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2284	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2285	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);
2286
2287	Tokens = annotate(Code: "#define FOO Foo::\n"
2288	"FOO Foo();");
2289	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2290	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_FunctionDeclarationName);
2291	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);
2292
2293	Tokens = annotate(Code: "struct Foo {\n"
2294	" Bar (*func)();\n"
2295	"};");
2296	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2297	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
2298	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionTypeLParen);
2299
2300	Tokens = annotate(Code: "void instanceof();");
2301	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2302	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2303	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
2304
2305	Tokens = annotate(Code: "#define FUNC(foo, bar, baz) \\\n"
2306	" auto foo##bar##baz() -> Type {}");
2307	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
2308	EXPECT_TOKEN(Tokens[11], tok::identifier, TT_FunctionDeclarationName);
2309	EXPECT_TOKEN(Tokens[16], tok::l_paren, TT_FunctionDeclarationLParen);
2310	EXPECT_TOKEN(Tokens[18], tok::arrow, TT_TrailingReturnArrow);
2311
2312	Tokens = annotate(Code: "void __stdcall f();");
2313	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2314	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_FunctionDeclarationName);
2315	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);
2316
2317	Tokens = annotate(Code: "int iso_time(time_t);");
2318	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2319	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2320	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
2321
2322	auto Style = getLLVMStyle();
2323	Style.TypeNames.push_back(x: "MyType");
2324	Tokens = annotate(Code: "int iso_time(MyType);", Style);
2325	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2326	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2327	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
2328	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_TypeName);
2329	}
2330
2331	TEST_F(TokenAnnotatorTest, UnderstandsCtorAndDtorDeclNames) {
2332	auto Tokens = annotate(Code: "class Foo { public: Foo(); };");
2333	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2334	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_CtorDtorDeclName);
2335	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionDeclarationLParen);
2336
2337	Tokens = annotate(Code: "class Foo { public: ~Foo(); };");
2338	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2339	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_CtorDtorDeclName);
2340	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);
2341
2342	Tokens = annotate(Code: "struct Foo { [[deprecated]] Foo() {} };");
2343	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2344	EXPECT_TOKEN(Tokens[8], tok::identifier, TT_CtorDtorDeclName);
2345	EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_FunctionDeclarationLParen);
2346	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_FunctionLBrace);
2347
2348	Tokens = annotate(Code: "struct Foo { [[deprecated]] ~Foo() {} };");
2349	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
2350	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_CtorDtorDeclName);
2351	EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_FunctionDeclarationLParen);
2352	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);
2353
2354	Tokens = annotate(Code: "struct Foo { Foo() [[deprecated]] {} };");
2355	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2356	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
2357	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);
2358	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_FunctionLBrace);
2359
2360	Tokens = annotate(Code: "struct Foo { ~Foo() [[deprecated]] {} };");
2361	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
2362	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_CtorDtorDeclName);
2363	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_FunctionDeclarationLParen);
2364	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);
2365
2366	Tokens = annotate(Code: "struct Foo { [[deprecated]] explicit Foo() {} };");
2367	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
2368	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_CtorDtorDeclName);
2369	EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_FunctionDeclarationLParen);
2370	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);
2371
2372	Tokens = annotate(Code: "struct Foo { virtual [[deprecated]] ~Foo() {} };");
2373	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
2374	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_CtorDtorDeclName);
2375	EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);
2376	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
2377
2378	Tokens = annotate(Code: "Foo::Foo() {}");
2379	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2380	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_CtorDtorDeclName);
2381	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);
2382	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);
2383
2384	Tokens = annotate(Code: "Foo::~Foo() {}");
2385	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2386	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
2387	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);
2388	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);
2389
2390	Tokens = annotate(Code: "struct Test {\n"
2391	" Test()\n"
2392	" : l([] {\n"
2393	" Short::foo();\n"
2394	" }) {}\n"
2395	"};");
2396	ASSERT_EQ(Tokens.size(), 25u) << Tokens;
2397	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
2398	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);
2399	EXPECT_TOKEN(Tokens[14], tok::identifier, TT_Unknown);
2400	}
2401
2402	TEST_F(TokenAnnotatorTest, UnderstandsC11GenericSelection) {
2403	auto Tokens = annotate(Code: "_Generic(x, int: 1, default: 0)");
2404	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2405	EXPECT_TOKEN(Tokens[0], tok::kw__Generic, TT_Unknown);
2406	EXPECT_TOKEN(Tokens[5], tok::colon, TT_GenericSelectionColon);
2407	EXPECT_TOKEN(Tokens[9], tok::colon, TT_GenericSelectionColon);
2408	}
2409
2410	TEST_F(TokenAnnotatorTest, UnderstandsTrailingReturnArrow) {
2411	auto Tokens = annotate(Code: "auto f() -> int;");
2412	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2413	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
2414
2415	Tokens = annotate(Code: "auto operator->() -> int;");
2416	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2417	EXPECT_TOKEN(Tokens[2], tok::arrow, TT_OverloadedOperator);
2418	EXPECT_TOKEN(Tokens[5], tok::arrow, TT_TrailingReturnArrow);
2419
2420	Tokens = annotate(Code: "auto operator++(int) -> int;");
2421	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2422	EXPECT_TOKEN(Tokens[6], tok::arrow, TT_TrailingReturnArrow);
2423
2424	Tokens = annotate(Code: "auto operator=() -> int;");
2425	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2426	EXPECT_TOKEN(Tokens[5], tok::arrow, TT_TrailingReturnArrow);
2427
2428	Tokens = annotate(Code: "auto operator=(int) -> int;");
2429	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2430	EXPECT_TOKEN(Tokens[6], tok::arrow, TT_TrailingReturnArrow);
2431
2432	Tokens = annotate(Code: "auto foo() -> auto { return Val; }");
2433	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2434	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
2435
2436	Tokens = annotate(Code: "struct S { auto bar() const -> int; };");
2437	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2438	EXPECT_TOKEN(Tokens[8], tok::arrow, TT_TrailingReturnArrow);
2439
2440	// Not trailing return arrows
2441	Tokens = annotate(Code: "auto a = b->c;");
2442	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2443	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);
2444
2445	Tokens = annotate(Code: "auto a = (b)->c;");
2446	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2447	EXPECT_TOKEN(Tokens[6], tok::arrow, TT_Unknown);
2448
2449	Tokens = annotate(Code: "auto a = b()->c;");
2450	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2451	EXPECT_TOKEN(Tokens[6], tok::arrow, TT_Unknown);
2452
2453	Tokens = annotate(Code: "auto a = b->c();");
2454	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2455	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);
2456
2457	Tokens = annotate(Code: "decltype(auto) a = b()->c;");
2458	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2459	EXPECT_TOKEN(Tokens[9], tok::arrow, TT_Unknown);
2460
2461	Tokens = annotate(Code: "void f() { auto a = b->c(); }");
2462	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2463	EXPECT_TOKEN(Tokens[9], tok::arrow, TT_Unknown);
2464
2465	Tokens = annotate(Code: "void f() { auto a = b()->c; }");
2466	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2467	EXPECT_TOKEN(Tokens[11], tok::arrow, TT_Unknown);
2468
2469	Tokens = annotate(Code: "#define P(ptr) auto p = (ptr)->p");
2470	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2471	EXPECT_TOKEN(Tokens[12], tok::arrow, TT_Unknown);
2472
2473	Tokens = annotate(Code: "void f() FOO(foo->bar);");
2474	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2475	EXPECT_TOKEN(Tokens[7], tok::arrow, TT_Unknown);
2476
2477	Tokens = annotate(Code: "__attribute__((cold)) C() : Base(obj->func()) {}");
2478	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
2479	EXPECT_TOKEN(Tokens[13], tok::arrow, TT_Unknown);
2480
2481	Tokens = annotate(Code: "Class<Type>{foo}->func(arg);");
2482	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2483	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_Unknown); // Not FunctionLBrace
2484	EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);
2485	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
2486	EXPECT_TOKEN(Tokens[7], tok::arrow, TT_Unknown);
2487
2488	auto Style = getLLVMStyle();
2489	Style.StatementAttributeLikeMacros.push_back(x: "emit");
2490	Tokens = annotate(Code: "emit foo()->bar;", Style);
2491	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2492	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_StatementAttributeLikeMacro);
2493	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);
2494
2495	// Mixed
2496	Tokens = annotate(Code: "auto f() -> int { auto a = b()->c; }");
2497	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
2498	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
2499	EXPECT_TOKEN(Tokens[13], tok::arrow, TT_Unknown);
2500	}
2501
2502	TEST_F(TokenAnnotatorTest, UnderstandHashInMacro) {
2503	auto Tokens = annotate(Code: "#define Foo(Bar) \\\n"
2504	" { \\\n"
2505	" #Bar \\\n"
2506	" }");
2507	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2508	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
2509	EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
2510
2511	Tokens = annotate(Code: "#define Foo(Bar) \\\n"
2512	" { #Bar }");
2513	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2514	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
2515	EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
2516
2517	Tokens = annotate(Code: "#define FOO(typeName, realClass) \\\n"
2518	" {#typeName, foo<Foo>(new foo<realClass>(#typeName))}");
2519	ASSERT_EQ(Tokens.size(), 29u) << Tokens;
2520	EXPECT_BRACE_KIND(Tokens[8], BK_BracedInit);
2521	EXPECT_BRACE_KIND(Tokens[27], BK_BracedInit);
2522	}
2523
2524	TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacros) {
2525	// '__attribute__' has special handling.
2526	auto Tokens = annotate(Code: "__attribute__((X)) void Foo(void);");
2527	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2528	EXPECT_TOKEN(Tokens[0], tok::kw___attribute, TT_Unknown);
2529	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
2530	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);
2531
2532	// Generic macro has no special handling in this location.
2533	Tokens = annotate(Code: "A(X) void Foo(void);");
2534	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2535	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2536	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_Unknown);
2537
2538	// Add a custom AttributeMacro. Test that it has the same behavior.
2539	FormatStyle Style = getLLVMStyle();
2540	Style.AttributeMacros.push_back(x: "A");
2541
2542	// An "AttributeMacro" gets annotated like '__attribute__'.
2543	Tokens = annotate(Code: "A(X) void Foo(void);", Style);
2544	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2545	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_AttributeMacro);
2546	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
2547	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);
2548	}
2549
2550	TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCDecl) {
2551	// '__attribute__' has special handling.
2552	auto Tokens = annotate(Code: "__attribute__((X)) @interface Foo");
2553	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2554	EXPECT_TOKEN(Tokens[0], tok::kw___attribute, TT_Unknown);
2555	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
2556	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);
2557
2558	// Generic macro has no special handling in this location.
2559	Tokens = annotate(Code: "A(X) @interface Foo");
2560	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2561	// Note: Don't check token-type as a random token in this position is hard to
2562	// reason about.
2563	EXPECT_TOKEN_KIND(Tokens[0], tok::identifier);
2564	EXPECT_TOKEN_KIND(Tokens[1], tok::l_paren);
2565
2566	// Add a custom AttributeMacro. Test that it has the same behavior.
2567	FormatStyle Style = getLLVMStyle();
2568	Style.AttributeMacros.push_back(x: "A");
2569
2570	// An "AttributeMacro" gets annotated like '__attribute__'.
2571	Tokens = annotate(Code: "A(X) @interface Foo", Style);
2572	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2573	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_AttributeMacro);
2574	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
2575	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);
2576	}
2577
2578	TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCMethodDecl) {
2579	// '__attribute__' has special handling.
2580	auto Tokens = annotate(Code: "- (id)init __attribute__((X));");
2581	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2582	EXPECT_TOKEN(Tokens[5], tok::kw___attribute, TT_Unknown);
2583	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_AttributeLParen);
2584	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_AttributeRParen);
2585
2586	// Generic macro has no special handling in this location.
2587	Tokens = annotate(Code: "- (id)init A(X);");
2588	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2589	// Note: Don't check token-type as a random token in this position is hard to
2590	// reason about.
2591	EXPECT_TOKEN_KIND(Tokens[5], tok::identifier);
2592	EXPECT_TOKEN_KIND(Tokens[6], tok::l_paren);
2593
2594	// Add a custom AttributeMacro. Test that it has the same behavior.
2595	FormatStyle Style = getLLVMStyle();
2596	Style.AttributeMacros.push_back(x: "A");
2597
2598	// An "AttributeMacro" gets annotated like '__attribute__'.
2599	Tokens = annotate(Code: "- (id)init A(X);", Style);
2600	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2601	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_AttributeMacro);
2602	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_AttributeLParen);
2603	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_AttributeRParen);
2604	}
2605
2606	TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCProperty) {
2607	// '__attribute__' has special handling.
2608	auto Tokens = annotate(Code: "@property(weak) id delegate __attribute__((X));");
2609	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2610	EXPECT_TOKEN(Tokens[7], tok::kw___attribute, TT_Unknown);
2611	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_AttributeLParen);
2612	EXPECT_TOKEN(Tokens[12], tok::r_paren, TT_AttributeRParen);
2613
2614	// Generic macro has no special handling in this location.
2615	Tokens = annotate(Code: "@property(weak) id delegate A(X);");
2616	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2617	// Note: Don't check token-type as a random token in this position is hard to
2618	// reason about.
2619	EXPECT_TOKEN_KIND(Tokens[7], tok::identifier);
2620	EXPECT_TOKEN_KIND(Tokens[8], tok::l_paren);
2621
2622	// Add a custom AttributeMacro. Test that it has the same behavior.
2623	FormatStyle Style = getLLVMStyle();
2624	Style.AttributeMacros.push_back(x: "A");
2625
2626	// An "AttributeMacro" gets annotated like '__attribute__'.
2627	Tokens = annotate(Code: "@property(weak) id delegate A(X);", Style);
2628	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2629	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_AttributeMacro);
2630	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_AttributeLParen);
2631	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_AttributeRParen);
2632	}
2633
2634	TEST_F(TokenAnnotatorTest, UnderstandsVerilogOperators) {
2635	auto Annotate = [this](StringRef Code) {
2636	return annotate(Code, Style: getLLVMStyle(Language: FormatStyle::LK_Verilog));
2637	};
2638	// Test that unary operators get labeled as such and that operators like '++'
2639	// don't get split.
2640	tok::TokenKind Unary[] = {tok::plus, tok::minus, tok::exclaim,
2641	tok::tilde, tok::amp, tok::pipe,
2642	tok::caret, tok::plusplus, tok::minusminus};
2643	for (auto Kind : Unary) {
2644	auto Tokens =
2645	Annotate(std::string("x = ") + tok::getPunctuatorSpelling(Kind) + "x;");
2646	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2647	EXPECT_TOKEN(Tokens[2], Kind, TT_UnaryOperator);
2648	}
2649	// Operators formed by joining two operators like '^~'. For some of these
2650	// joined operators, we don't have a separate type, so we only test for their
2651	// precedence.
2652	std::pair<prec::Level, std::string> JoinedBinary[] = {
2653	{prec::Comma, "->"}, {prec::Comma, "<->"},
2654	{prec::Assignment, "+="}, {prec::Assignment, "-="},
2655	{prec::Assignment, "*="}, {prec::Assignment, "/="},
2656	{prec::Assignment, "%="}, {prec::Assignment, "&="},
2657	{prec::Assignment, "^="}, {prec::Assignment, "<<="},
2658	{prec::Assignment, ">>="}, {prec::Assignment, "<<<="},
2659	{prec::Assignment, ">>>="}, {prec::LogicalOr, "||"},
2660	{prec::LogicalAnd, "&&"}, {prec::Equality, "=="},
2661	{prec::Equality, "!="}, {prec::Equality, "==="},
2662	{prec::Equality, "!=="}, {prec::Equality, "==?"},
2663	{prec::Equality, "!=?"}, {prec::ExclusiveOr, "~^"},
2664	{prec::ExclusiveOr, "^~"},
2665	};
2666	for (auto Operator : JoinedBinary) {
2667	auto Tokens = Annotate(std::string("x = x ") + Operator.second + " x;");
2668	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2669	EXPECT_TOKEN_TYPE(Tokens[3], TT_BinaryOperator);
2670	EXPECT_TOKEN_PRECEDENCE(Tokens[3], Operator.first);
2671	}
2672	// '~^' and '^~' can be unary as well as binary operators.
2673	auto Tokens = Annotate("x = ~^x;");
2674	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2675	EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator);
2676	Tokens = Annotate("x = ^~x;");
2677	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2678	EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator);
2679	// The unary operators '~&' and '~|' can only be unary operators. The current
2680	// implementation treats each of them as separate unary '~' and '&' or '|'
2681	// operators, which is enough for formatting purposes. In FormatTestVerilog,
2682	// there is a test that there is no space in between. And even if a new line
2683	// is inserted between the '~' and '|', the semantic meaning is the same as
2684	// the joined operator, so the CanBreakBefore property doesn't need to be
2685	// false for the second operator.
2686	Tokens = Annotate("x = ~&x;");
2687	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2688	EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator);
2689	EXPECT_TOKEN(Tokens[3], tok::amp, TT_UnaryOperator);
2690	Tokens = Annotate("x = ~|x;");
2691	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2692	EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator);
2693	EXPECT_TOKEN(Tokens[3], tok::pipe, TT_UnaryOperator);
2694	// Test for block label colons.
2695	Tokens = Annotate("begin : x\n"
2696	"end : x");
2697	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2698	EXPECT_TOKEN(Tokens[1], tok::colon, TT_VerilogBlockLabelColon);
2699	EXPECT_TOKEN(Tokens[4], tok::colon, TT_VerilogBlockLabelColon);
2700	// Test that the dimension colon is annotated correctly.
2701	Tokens = Annotate("var [1 : 0] x;");
2702	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2703	EXPECT_TOKEN(Tokens[3], tok::colon, TT_BitFieldColon);
2704	Tokens = Annotate("extern function [1 : 0] x;");
2705	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2706	EXPECT_TOKEN(Tokens[4], tok::colon, TT_BitFieldColon);
2707	Tokens = Annotate("module test\n"
2708	" (input wire [7 : 0] a[7 : 0]);\n"
2709	"endmodule");
2710	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
2711	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_VerilogDimensionedTypeName);
2712	EXPECT_TOKEN(Tokens[7], tok::colon, TT_BitFieldColon);
2713	EXPECT_TOKEN(Tokens[13], tok::colon, TT_BitFieldColon);
2714	// Test case labels and ternary operators.
2715	Tokens = Annotate("case (x)\n"
2716	" x:\n"
2717	" x;\n"
2718	"endcase");
2719	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2720	EXPECT_TOKEN(Tokens[5], tok::colon, TT_CaseLabelColon);
2721	Tokens = Annotate("case (x)\n"
2722	" x ? x : x:\n"
2723	" x;\n"
2724	"endcase");
2725	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2726	EXPECT_TOKEN(Tokens[5], tok::question, TT_ConditionalExpr);
2727	EXPECT_TOKEN(Tokens[7], tok::colon, TT_ConditionalExpr);
2728	EXPECT_TOKEN(Tokens[9], tok::colon, TT_CaseLabelColon);
2729	// Non-blocking assignments.
2730	Tokens = Annotate("a <= b;");
2731	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
2732	EXPECT_TOKEN(Tokens[1], tok::lessequal, TT_BinaryOperator);
2733	EXPECT_TOKEN_PRECEDENCE(Tokens[1], prec::Assignment);
2734	Tokens = Annotate("if (a <= b) break;");
2735	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2736	EXPECT_TOKEN(Tokens[3], tok::lessequal, TT_BinaryOperator);
2737	EXPECT_TOKEN_PRECEDENCE(Tokens[3], prec::Relational);
2738	Tokens = Annotate("a <= b <= a;");
2739	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2740	EXPECT_TOKEN(Tokens[1], tok::lessequal, TT_BinaryOperator);
2741	EXPECT_TOKEN_PRECEDENCE(Tokens[1], prec::Assignment);
2742	EXPECT_TOKEN(Tokens[3], tok::lessequal, TT_BinaryOperator);
2743	EXPECT_TOKEN_PRECEDENCE(Tokens[3], prec::Relational);
2744
2745	// Port lists in module instantiation.
2746	Tokens = Annotate("module_x instance_1(port_1), instance_2(port_2);");
2747	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2748	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_VerilogInstancePortLParen);
2749	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_VerilogInstancePortLParen);
2750	Tokens = Annotate("module_x #(parameter) instance_1(port_1), "
2751	"instance_2(port_2);");
2752	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2753	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_VerilogInstancePortLParen);
2754	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_VerilogInstancePortLParen);
2755	EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_VerilogInstancePortLParen);
2756
2757	// Condition parentheses.
2758	Tokens = Annotate("assert (x);");
2759	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2760	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2761	Tokens = Annotate("assert #0 (x);");
2762	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2763	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_ConditionLParen);
2764	Tokens = Annotate("assert final (x);");
2765	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2766	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);
2767	Tokens = Annotate("foreach (x[x]) continue;");
2768	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2769	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2770	Tokens = Annotate("repeat (x[x]) continue;");
2771	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2772	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2773	Tokens = Annotate("case (x) endcase;");
2774	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2775	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2776
2777	// Sensitivity list. The TT_Unknown type is clearly not binding for the
2778	// future, please adapt if those tokens get annotated. This test is only here
2779	// to prevent the comma from being annotated as TT_VerilogInstancePortComma.
2780	Tokens = Annotate("always @(posedge x, posedge y);");
2781	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2782	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_Unknown);
2783	EXPECT_TOKEN(Tokens[5], tok::comma, TT_Unknown);
2784	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
2785
2786	// String literals in concatenation.
2787	Tokens = Annotate("x = {\"\"};");
2788	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2789	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_StringInConcatenation);
2790	Tokens = Annotate("x = {\"\", \"\"};");
2791	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2792	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_StringInConcatenation);
2793	EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_StringInConcatenation);
2794	Tokens = Annotate("x = '{{\"\"}};");
2795	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2796	EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_StringInConcatenation);
2797	// Cases where the string should not be annotated that type. Fix the
2798	// `TT_Unknown` if needed in the future.
2799	Tokens = Annotate("x = {\"\" == \"\"};");
2800	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2801	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_Unknown);
2802	EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_Unknown);
2803	Tokens = Annotate("x = {(\"\")};");
2804	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2805	EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_Unknown);
2806	Tokens = Annotate("x = '{\"\"};");
2807	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2808	EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_Unknown);
2809
2810	// Module headers.
2811	Tokens = Annotate("module x();\n"
2812	"endmodule");
2813	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2814	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_VerilogMultiLineListLParen);
2815	Tokens = Annotate("function automatic `x x();\n"
2816	"endmodule");
2817	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2818	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_VerilogMultiLineListLParen);
2819	Tokens = Annotate("function automatic x``x x();\n"
2820	"endmodule");
2821	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2822	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_VerilogMultiLineListLParen);
2823	Tokens = Annotate("function automatic x::x x();\n"
2824	"endmodule");
2825	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2826	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_VerilogMultiLineListLParen);
2827
2828	// Escaped identifiers.
2829	Tokens = Annotate(R"(\busa+index)");
2830	ASSERT_EQ(Tokens.size(), 2u) << Tokens;
2831	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2832	Tokens = Annotate(R"(\busa+index ;)");
2833	ASSERT_EQ(Tokens.size(), 3u) << Tokens;
2834	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2835	EXPECT_EQ(Tokens[0]->TokenText, R"(\busa+index)");
2836	EXPECT_TOKEN(Tokens[1], tok::semi, TT_Unknown);
2837	Tokens = Annotate(R"(\busa+index
2838	;)");
2839	ASSERT_EQ(Tokens.size(), 3u) << Tokens;
2840	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2841	EXPECT_TOKEN(Tokens[1], tok::semi, TT_Unknown);
2842	// The escaped identifier can be broken by an escaped newline. The result is
2843	// still 1 identifier.
2844	Tokens = Annotate(R"(\busa+index\
2845	+
2846	;)");
2847	ASSERT_EQ(Tokens.size(), 3u) << Tokens;
2848	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2849	EXPECT_EQ(Tokens[0]->TokenText, R"(\busa+index\
2850	+)");
2851	EXPECT_TOKEN(Tokens[1], tok::semi, TT_Unknown);
2852	// An escaped newline should not be treated as an escaped identifier.
2853	Tokens = Annotate("\\\n");
2854	ASSERT_EQ(Tokens.size(), 1u) << Tokens;
2855	EXPECT_TOKEN(Tokens[0], tok::eof, TT_Unknown);
2856	// Macros.
2857	Tokens = Annotate("`define x x``x");
2858	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2859	EXPECT_TOKEN(Tokens[0], tok::hash, TT_Unknown);
2860	EXPECT_TOKEN(Tokens[4], tok::hashhash, TT_Unknown);
2861	}
2862
2863	TEST_F(TokenAnnotatorTest, UnderstandTableGenTokens) {
2864	auto Style = getLLVMStyle(Language: FormatStyle::LK_TableGen);
2865	ASSERT_TRUE(Style.isTableGen());
2866
2867	TestLexer Lexer(Allocator, Buffers, Style);
2868	AdditionalKeywords Keywords(Lexer.IdentTable);
2869	auto Annotate = [&Lexer](StringRef Code) { return Lexer.annotate(Code); };
2870
2871	// Additional keywords representation test.
2872	auto Tokens = Annotate("def foo : Bar<1>;");
2873	ASSERT_TRUE(Keywords.isTableGenKeyword(*Tokens[0]));
2874	ASSERT_TRUE(Keywords.isTableGenDefinition(*Tokens[0]));
2875	ASSERT_TRUE(Tokens[0]->is(Keywords.kw_def));
2876	ASSERT_TRUE(Tokens[1]->is(TT_StartOfName));
2877
2878	// Code, the multiline string token.
2879	Tokens = Annotate("[{ code is multiline string }]");
2880	ASSERT_EQ(Tokens.size(), 2u) << Tokens;
2881	EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString);
2882	EXPECT_FALSE(Tokens[0]->IsMultiline);
2883	// Case with multiple lines.
2884	Tokens = Annotate("[{ It can break\n"
2885	" across lines and the line breaks\n"
2886	" are retained in \n"
2887	" the string. }]");
2888	ASSERT_EQ(Tokens.size(), 2u) << Tokens;
2889	EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString);
2890	EXPECT_EQ(Tokens[0]->ColumnWidth, sizeof("[{ It can break\n") - 1);
2891	EXPECT_TRUE(Tokens[0]->IsMultiline);
2892	EXPECT_EQ(Tokens[0]->LastLineColumnWidth, sizeof(" the string. }]") - 1);
2893
2894	// Numeric literals.
2895	Tokens = Annotate("1234");
2896	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2897	Tokens = Annotate("-1");
2898	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2899	Tokens = Annotate("+1234");
2900	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2901	Tokens = Annotate("0b0110");
2902	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2903	Tokens = Annotate("0x1abC");
2904	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2905
2906	// Identifier tokens. In TableGen, identifiers can begin with a number.
2907	// In ambiguous cases, the lexer tries to lex it as a number.
2908	// Even if the try fails, it does not fall back to identifier lexing and
2909	// regard as an error.
2910	// The ambiguity is not documented. The result of those tests are based on the
2911	// implementation of llvm::TGLexer::LexToken.
2912	// This is invalid syntax of number, but not an identifier.
2913	Tokens = Annotate("0x1234x");
2914	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2915	Tokens = Annotate("identifier");
2916	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2917	// Identifier beginning with a number.
2918	Tokens = Annotate("0x");
2919	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2920	Tokens = Annotate("2dVector");
2921	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2922	Tokens = Annotate("01234Vector");
2923	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2924
2925	// Structured statements.
2926	Tokens = Annotate("class Foo {}");
2927	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
2928	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_FunctionLBrace);
2929	Tokens = Annotate("def Def: Foo {}");
2930	EXPECT_TOKEN(Tokens[2], tok::colon, TT_InheritanceColon);
2931	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
2932	Tokens = Annotate("if cond then {} else {}");
2933	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_ControlStatementLBrace);
2934	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_ElseLBrace);
2935	Tokens = Annotate("defset Foo Def2 = {}");
2936	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
2937
2938	// Bang Operators.
2939	Tokens = Annotate("!foreach");
2940	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenBangOperator);
2941	Tokens = Annotate("!if");
2942	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenBangOperator);
2943	Tokens = Annotate("!cond");
2944	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenCondOperator);
2945
2946	// The paste operator should not be treated as a preprocessor directive even
2947	// if it is on a separate line.
2948	Tokens = Annotate("def x\n"
2949	"#embed {}");
2950	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2951	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
2952	EXPECT_EQ(Tokens[1]->Next, Tokens[2]);
2953	Tokens = Annotate("def x\n"
2954	"#define x\n"
2955	"#embed {}");
2956	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2957	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
2958	EXPECT_EQ(Tokens[1]->Next, Tokens[5]);
2959	Tokens = Annotate("def x\n"
2960	"#ifdef x\n"
2961	"#else\n"
2962	"#endif\n"
2963	"#embed {}");
2964	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2965	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
2966	EXPECT_EQ(Tokens[1]->Next, Tokens[9]);
2967
2968	auto AnnotateValue = [this, &Style](StringRef Code) {
2969	// Values are annotated only in specific context.
2970	auto Result = annotate(Code: ("def X { let V = " + Code + "; }").str(), Style);
2971	return decltype(Result){Result.begin() + 6, Result.end() - 3};
2972	};
2973	// Both of bang/cond operators.
2974	Tokens = AnnotateValue("!cond(!eq(x, 0): 1, true: x)");
2975	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2976	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenCondOperator);
2977	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TableGenBangOperator);
2978	EXPECT_TOKEN(Tokens[8], tok::colon, TT_TableGenCondOperatorColon);
2979	EXPECT_TOKEN(Tokens[10], tok::comma, TT_TableGenCondOperatorComma);
2980	EXPECT_TOKEN(Tokens[12], tok::colon, TT_TableGenCondOperatorColon);
2981	// DAGArg values with operator identifier
2982	Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
2983	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2984	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpener);
2985	EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColon);
2986	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown); // $src1
2987	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListComma);
2988	EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColon);
2989	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2990	// List literal
2991	Tokens = AnnotateValue("[1, 2, 3]");
2992	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2993	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_TableGenListOpener);
2994	EXPECT_TOKEN(Tokens[6], tok::r_square, TT_TableGenListCloser);
2995	// Suffixes of values
2996	Tokens = AnnotateValue("valid.field");
2997	ASSERT_EQ(Tokens.size(), 3u) << Tokens;
2998	EXPECT_TOKEN(Tokens[1], tok::period, TT_TableGenValueSuffix);
2999	// Code
3000	Tokens = AnnotateValue("[{ code is multiline string }]");
3001	ASSERT_EQ(Tokens.size(), 1u) << Tokens;
3002	EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString);
3003
3004	// The definition
3005	Tokens = annotate(Code: "def Def : Parent<Child> {}", Style);
3006	ASSERT_EQ(Tokens.size(), 10u) << Tokens; // This contains eof.
3007	// We use inheritance colon and function brace. They are enough.
3008	EXPECT_TOKEN(Tokens[2], tok::colon, TT_InheritanceColon);
3009	EXPECT_TOKEN(Tokens[4], tok::less, TT_TemplateOpener);
3010	EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
3011	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
3012
3013	// DAGArg breaking options. They use different token types depending on what
3014	// is specified.
3015	Style.TableGenBreakInsideDAGArg = FormatStyle::DAS_BreakElements;
3016
3017	// When TableGenBreakInsideDAGArg is DAS_BreakElements and
3018	// TableGenBreakingDAGArgOperators is not specified, it makes all the DAGArg
3019	// elements to have line break.
3020	Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
3021	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3022	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
3023	EXPECT_TOKEN(Tokens[1], tok::identifier,
3024	TT_TableGenDAGArgOperatorID); // ins
3025	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
3026	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
3027
3028	Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
3029	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3030	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
3031	EXPECT_TOKEN(Tokens[1], tok::identifier,
3032	TT_TableGenDAGArgOperatorID); // other
3033	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
3034	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
3035
3036	// For non-identifier operators, breaks after the operator.
3037	Tokens = AnnotateValue("(!cast<Type>(\"Name\") type1:$src1, type2:$src2)");
3038	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
3039	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
3040	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_TableGenDAGArgOperatorToBreak);
3041	EXPECT_TOKEN(Tokens[11], tok::comma, TT_TableGenDAGArgListCommaToBreak);
3042	EXPECT_TOKEN(Tokens[15], tok::r_paren, TT_TableGenDAGArgCloser);
3043
3044	Style.TableGenBreakInsideDAGArg = FormatStyle::DAS_BreakAll;
3045
3046	// When TableGenBreakInsideDAGArg is DAS_BreakAll and
3047	// TableGenBreakingDAGArgOperators is not specified, it makes all the DAGArg
3048	// to have line break inside it.
3049	Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
3050	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3051	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
3052	EXPECT_TOKEN(Tokens[1], tok::identifier,
3053	TT_TableGenDAGArgOperatorToBreak); // ins
3054	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
3055	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
3056
3057	Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
3058	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3059	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
3060	EXPECT_TOKEN(Tokens[1], tok::identifier,
3061	TT_TableGenDAGArgOperatorToBreak); // other
3062	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
3063	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
3064
3065	// If TableGenBreakingDAGArgOperators is specified, it is limited to the
3066	// specified operators.
3067	Style.TableGenBreakingDAGArgOperators = {"ins", "outs"};
3068	Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
3069	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3070	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
3071	EXPECT_TOKEN(Tokens[1], tok::identifier,
3072	TT_TableGenDAGArgOperatorToBreak); // ins
3073	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
3074	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
3075
3076	Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
3077	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3078	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpener);
3079	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other
3080	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListComma);
3081	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
3082
3083	// If TableGenBreakingDAGArgOperators is enabled, it uses
3084	// TT_TableGenDAGArgListColonToAlign to annotate the colon to align.
3085	Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled = true;
3086	Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
3087	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3088	EXPECT_TOKEN(Tokens[1], tok::identifier,
3089	TT_TableGenDAGArgOperatorToBreak); // ins
3090	EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColonToAlign);
3091	EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColonToAlign);
3092
3093	Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
3094	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3095	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other
3096	EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColon);
3097	EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColon);
3098	}
3099
3100	TEST_F(TokenAnnotatorTest, UnderstandConstructors) {
3101	auto Tokens = annotate(Code: "Class::Class() : BaseClass(), Member() {}");
3102
3103	// The TT_Unknown is clearly not binding for the future, please adapt if those
3104	// tokens get annotated.
3105	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
3106	EXPECT_TOKEN(Tokens[5], tok::colon, TT_CtorInitializerColon);
3107	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
3108	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_Unknown);
3109	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
3110	EXPECT_TOKEN(Tokens[9], tok::comma, TT_CtorInitializerComma);
3111	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_Unknown);
3112	EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_Unknown);
3113	EXPECT_TOKEN(Tokens[12], tok::r_paren, TT_Unknown);
3114	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
3115	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3116
3117	Tokens = annotate(Code: "Class::Class() : BaseClass{}, Member{} {}");
3118	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
3119	EXPECT_TOKEN(Tokens[5], tok::colon, TT_CtorInitializerColon);
3120	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
3121	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_Unknown);
3122	EXPECT_TOKEN(Tokens[8], tok::r_brace, TT_Unknown);
3123	EXPECT_TOKEN(Tokens[9], tok::comma, TT_CtorInitializerComma);
3124	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_Unknown);
3125	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_Unknown);
3126	EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_Unknown);
3127	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
3128	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3129
3130	Tokens = annotate(Code: "class Class {\n"
3131	" Class() : BaseClass() {\n"
3132	"#if 0\n"
3133	" // comment\n"
3134	"#endif\n"
3135	" }\n"
3136	" Class f();\n"
3137	"}");
3138	ASSERT_EQ(Tokens.size(), 25u) << Tokens;
3139	EXPECT_TOKEN(Tokens[6], tok::colon, TT_CtorInitializerColon);
3140	EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_FunctionLBrace);
3141	EXPECT_BRACE_KIND(Tokens[10], BK_Block);
3142	}
3143
3144	TEST_F(TokenAnnotatorTest, UnderstandsConditionParens) {
3145	auto Tokens = annotate(Code: "if (x) {}");
3146	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3147	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
3148	Tokens = annotate(Code: "if constexpr (x) {}");
3149	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3150	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);
3151	Tokens = annotate(Code: "if CONSTEXPR (x) {}");
3152	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3153	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);
3154	Tokens = annotate(Code: "if (x) {} else if (x) {}");
3155	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3156	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
3157	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_ConditionLParen);
3158	}
3159
3160	TEST_F(TokenAnnotatorTest, CSharpNullableTypes) {
3161	FormatStyle Style = getGoogleStyle(Language: FormatStyle::LK_CSharp);
3162
3163	auto Tokens = annotate(Code: "int? a;", Style);
3164	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
3165	EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
3166
3167	Tokens = annotate(Code: "int? a = 1;", Style);
3168	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3169	EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
3170
3171	Tokens = annotate(Code: "int?)", Style);
3172	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
3173	EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
3174
3175	Tokens = annotate(Code: "int?>", Style);
3176	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
3177	EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
3178
3179	Tokens = annotate(Code: "{\n"
3180	" int? a;\n"
3181	" if (b is int?)\n"
3182	" f();\n"
3183	" var foo = A<Foo?>();\n"
3184	"}",
3185	Style);
3186	ASSERT_EQ(Tokens.size(), 29u) << Tokens;
3187	EXPECT_TOKEN(Tokens[2], tok::question, TT_CSharpNullable);
3188	EXPECT_TOKEN(Tokens[10], tok::question, TT_CSharpNullable);
3189	EXPECT_TOKEN(Tokens[20], tok::less, TT_TemplateOpener);
3190	EXPECT_TOKEN(Tokens[22], tok::question, TT_CSharpNullable);
3191
3192	Tokens = annotate(Code: "cond? id : id2", Style);
3193	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3194	EXPECT_TOKEN(Tokens[1], tok::question, TT_ConditionalExpr);
3195
3196	Tokens = annotate(Code: "cond ? cond2 ? : id1 : id2", Style);
3197	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3198	EXPECT_TOKEN(Tokens[1], tok::question, TT_ConditionalExpr);
3199	}
3200
3201	TEST_F(TokenAnnotatorTest, CSharpGenericTypeConstraint) {
3202	auto Tokens = annotate(Code: "namespace A {\n"
3203	" delegate T MyDelegate<T>()\n"
3204	" where T : new();\n"
3205	"}",
3206	Style: getGoogleStyle(Language: FormatStyle::LK_CSharp));
3207	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
3208	EXPECT_TOKEN(Tokens[18], tok::r_brace, TT_NamespaceRBrace);
3209	}
3210
3211	TEST_F(TokenAnnotatorTest, CSharpNewModifier) {
3212	auto Tokens = annotate(Code: "new public class NestedC {\n"
3213	" public int x = 100;\n"
3214	"}",
3215	Style: getGoogleStyle(Language: FormatStyle::LK_CSharp));
3216	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
3217	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_ClassHeadName);
3218	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ClassLBrace);
3219	EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_ClassRBrace);
3220	}
3221
3222	TEST_F(TokenAnnotatorTest, UnderstandsLabels) {
3223	auto Tokens = annotate(Code: "{ x: break; }");
3224	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3225	EXPECT_TOKEN(Tokens[2], tok::colon, TT_GotoLabelColon);
3226
3227	Tokens = annotate(Code: "{ case x: break; }");
3228	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3229	EXPECT_TOKEN(Tokens[3], tok::colon, TT_CaseLabelColon);
3230
3231	Tokens = annotate(Code: "{ x: { break; } }");
3232	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3233	EXPECT_TOKEN(Tokens[2], tok::colon, TT_GotoLabelColon);
3234
3235	Tokens = annotate(Code: "{ case x: { break; } }");
3236	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3237	EXPECT_TOKEN(Tokens[3], tok::colon, TT_CaseLabelColon);
3238
3239	Tokens = annotate(Code: "#define FOO label:");
3240	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3241	EXPECT_TOKEN(Tokens[4], tok::colon, TT_GotoLabelColon);
3242
3243	Tokens = annotate(Code: "#define FOO \\\n"
3244	"label: \\\n"
3245	" break;");
3246	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3247	EXPECT_TOKEN(Tokens[4], tok::colon, TT_GotoLabelColon);
3248	}
3249
3250	TEST_F(TokenAnnotatorTest, UnderstandsNestedBlocks) {
3251	// The closing braces are not annotated. It doesn't seem to cause a problem.
3252	// So we only test for the opening braces.
3253	auto Tokens = annotate(Code: "{\n"
3254	" {\n"
3255	" { int a = 0; }\n"
3256	" }\n"
3257	" {}\n"
3258	"}");
3259	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3260	EXPECT_BRACE_KIND(Tokens[0], BK_Block);
3261	EXPECT_BRACE_KIND(Tokens[1], BK_Block);
3262	EXPECT_BRACE_KIND(Tokens[2], BK_Block);
3263	EXPECT_BRACE_KIND(Tokens[10], BK_Block);
3264	}
3265
3266	TEST_F(TokenAnnotatorTest, UnderstandDesignatedInitializers) {
3267	auto Tokens = annotate(Code: "SomeStruct { .a = 1 };");
3268	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3269	EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
3270	EXPECT_TOKEN(Tokens[2], tok::period, TT_DesignatedInitializerPeriod);
3271
3272	Tokens = annotate(Code: "SomeStruct { .a = 1, .b = 2 };");
3273	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3274	EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
3275	EXPECT_TOKEN(Tokens[2], tok::period, TT_DesignatedInitializerPeriod);
3276	EXPECT_TOKEN(Tokens[7], tok::period, TT_DesignatedInitializerPeriod);
3277
3278	Tokens = annotate(Code: "SomeStruct {\n"
3279	"#ifdef FOO\n"
3280	" .a = 1,\n"
3281	"#endif\n"
3282	" .b = 2\n"
3283	"};");
3284	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
3285	EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
3286	EXPECT_TOKEN(Tokens[5], tok::period, TT_DesignatedInitializerPeriod);
3287	EXPECT_TOKEN(Tokens[12], tok::period, TT_DesignatedInitializerPeriod);
3288
3289	Tokens = annotate(Code: "SomeStruct {\n"
3290	"#if defined FOO\n"
3291	" .a = 1,\n"
3292	"#endif\n"
3293	" .b = 2\n"
3294	"};");
3295	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
3296	EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
3297	EXPECT_TOKEN(Tokens[6], tok::period, TT_DesignatedInitializerPeriod);
3298	EXPECT_TOKEN(Tokens[13], tok::period, TT_DesignatedInitializerPeriod);
3299
3300	Tokens = annotate(Code: "Foo foo[] = {[0]{}};");
3301	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3302	EXPECT_TOKEN(Tokens[6], tok::l_square, TT_DesignatedInitializerLSquare);
3303	EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
3304	}
3305
3306	TEST_F(TokenAnnotatorTest, UnderstandsJavaScript) {
3307	auto Annotate = [this](StringRef Code) {
3308	return annotate(Code, Style: getLLVMStyle(Language: FormatStyle::LK_JavaScript));
3309	};
3310
3311	// Dictionary.
3312	auto Tokens = Annotate("var x = {'x' : 1, 'y' : 2};");
3313	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3314	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_DictLiteral);
3315	EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_SelectorName);
3316	EXPECT_TOKEN(Tokens[5], tok::colon, TT_DictLiteral);
3317	EXPECT_TOKEN(Tokens[8], tok::string_literal, TT_SelectorName);
3318	EXPECT_TOKEN(Tokens[9], tok::colon, TT_DictLiteral);
3319	// Change when we need to annotate these.
3320	EXPECT_BRACE_KIND(Tokens[3], BK_Unknown);
3321	EXPECT_BRACE_KIND(Tokens[11], BK_Unknown);
3322	EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_Unknown);
3323	}
3324
3325	TEST_F(TokenAnnotatorTest, UnderstandsAttributes) {
3326	auto Tokens = annotate(Code: "bool foo __attribute__((unused));");
3327	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3328	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
3329	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_AttributeLParen);
3330	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_Unknown);
3331	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_Unknown);
3332	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_AttributeRParen);
3333
3334	Tokens = annotate(Code: "bool foo __declspec(dllimport);");
3335	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3336	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_AttributeLParen);
3337	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);
3338
3339	Tokens = annotate(Code: "bool __attribute__((unused)) foo;");
3340	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3341	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_AttributeLParen);
3342	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_Unknown);
3343	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_Unknown);
3344	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_AttributeRParen);
3345	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_StartOfName);
3346
3347	Tokens = annotate(Code: "void __attribute__((x)) Foo();");
3348	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3349	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_AttributeLParen);
3350	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_Unknown);
3351	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_Unknown);
3352	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_AttributeRParen);
3353	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_FunctionDeclarationName);
3354	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_FunctionDeclarationLParen);
3355
3356	FormatStyle Style = getLLVMStyle();
3357	Style.AttributeMacros.push_back(x: "FOO");
3358	Tokens = annotate(Code: "bool foo FOO(unused);", Style);
3359	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3360	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_AttributeMacro);
3361	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_AttributeLParen);
3362	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);
3363	}
3364
3365	TEST_F(TokenAnnotatorTest, UnderstandsNullabilityAttributeMacros) {
3366	// Under Google style, handles the Abseil macro aliases for the Clang
3367	// nullability annotations.
3368	auto Style = getGoogleStyle(Language: FormatStyle::LK_Cpp);
3369	auto Tokens = annotate(Code: "x = (foo* absl_nullable)*v;", Style);
3370	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3371	EXPECT_TOKEN(Tokens[4], tok::star, TT_PointerOrReference);
3372	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_AttributeMacro);
3373	EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
3374
3375	Tokens = annotate(Code: "x = (foo* absl_nonnull)*v;", Style);
3376	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3377	EXPECT_TOKEN(Tokens[4], tok::star, TT_PointerOrReference);
3378	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_AttributeMacro);
3379	EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
3380
3381	Tokens = annotate(Code: "x = (foo* absl_nullability_unknown)*v;", Style);
3382	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3383	EXPECT_TOKEN(Tokens[4], tok::star, TT_PointerOrReference);
3384	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_AttributeMacro);
3385	EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
3386	}
3387
3388	TEST_F(TokenAnnotatorTest, UnderstandsControlStatements) {
3389	auto Tokens = annotate(Code: "while (true) {}");
3390	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3391	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);
3392	EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);
3393
3394	Tokens = annotate(Code: "for (;;) {}");
3395	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3396	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_ControlStatementLBrace);
3397	EXPECT_TOKEN(Tokens[6], tok::r_brace, TT_ControlStatementRBrace);
3398
3399	Tokens = annotate(Code: "do {} while (true);");
3400	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3401	EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_ControlStatementLBrace);
3402	EXPECT_TOKEN(Tokens[2], tok::r_brace, TT_ControlStatementRBrace);
3403
3404	Tokens = annotate(Code: "if (true) {} else if (false) {} else {}");
3405	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
3406	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);
3407	EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);
3408	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_ControlStatementLBrace);
3409	EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_ControlStatementRBrace);
3410	EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_ElseLBrace);
3411	EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_ElseRBrace);
3412
3413	Tokens = annotate(Code: "switch (foo) {}");
3414	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3415	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);
3416	EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);
3417	}
3418
3419	TEST_F(TokenAnnotatorTest, UnderstandsDoWhile) {
3420	auto Tokens = annotate(Code: "do { ++i; } while ( i > 5 );");
3421	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3422	EXPECT_TOKEN(Tokens[6], tok::kw_while, TT_DoWhile);
3423
3424	Tokens = annotate(Code: "do ++i; while ( i > 5 );");
3425	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3426	EXPECT_TOKEN(Tokens[4], tok::kw_while, TT_DoWhile);
3427	}
3428
3429	TEST_F(TokenAnnotatorTest, StartOfName) {
3430	auto Tokens = annotate(Code: "#pragma clang diagnostic push");
3431	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3432	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown);
3433	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
3434	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);
3435
3436	Tokens = annotate(Code: "#pragma clang diagnostic ignored \"-Wzero-length-array\"");
3437	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3438	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown);
3439	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
3440	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);
3441
3442	Tokens = annotate(Code: "#define FOO Foo foo");
3443	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3444	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown);
3445	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
3446	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_StartOfName);
3447
3448	Tokens = annotate(Code: "@interface NSCoder (TestCoder)");
3449	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3450	EXPECT_TOKEN(Tokens[0], tok::at, TT_ObjCDecl);
3451	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_StartOfName);
3452
3453	Tokens = annotate(Code: "class FOO BAR C {};");
3454	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3455	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown); // Not StartOfName
3456
3457	auto Style = getLLVMStyle();
3458	Style.StatementAttributeLikeMacros.push_back(x: "emit");
3459	Tokens = annotate(Code: "emit foo = 0;", Style);
3460	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3461	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_StatementAttributeLikeMacro);
3462	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);
3463	}
3464
3465	TEST_F(TokenAnnotatorTest, BraceKind) {
3466	auto Tokens = annotate(Code: "void f() {};");
3467	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3468	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3469	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3470	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
3471	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3472	EXPECT_BRACE_KIND(Tokens[5], BK_Block);
3473
3474	Tokens = annotate(Code: "class Foo<int> f() {}");
3475	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3476	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);
3477	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_FunctionDeclarationName);
3478	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionDeclarationLParen);
3479	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);
3480	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3481	EXPECT_BRACE_KIND(Tokens[9], BK_Block);
3482
3483	Tokens = annotate(Code: "template <typename T> class Foo<T> f() {}");
3484	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
3485	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
3486	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_FunctionDeclarationName);
3487	EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);
3488	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
3489	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3490	EXPECT_BRACE_KIND(Tokens[14], BK_Block);
3491
3492	Tokens = annotate(Code: "void f() override {};");
3493	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3494	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3495	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3496	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);
3497	EXPECT_BRACE_KIND(Tokens[5], BK_Block);
3498	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3499
3500	Tokens = annotate(Code: "void f() noexcept(false) {};");
3501	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3502	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3503	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3504	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);
3505	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3506	EXPECT_BRACE_KIND(Tokens[9], BK_Block);
3507
3508	Tokens = annotate(Code: "auto f() -> void {};");
3509	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3510	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3511	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3512	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);
3513	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3514	EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3515
3516	Tokens = annotate(Code: "void f() { /**/ };");
3517	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3518	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3519	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3520	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
3521	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3522	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3523
3524	Tokens = annotate(Code: "void f() { //\n"
3525	"};");
3526	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3527	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3528	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3529	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
3530	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3531	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3532
3533	Tokens = annotate(Code: "void f() {\n"
3534	" //\n"
3535	"};");
3536	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3537	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3538	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3539	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
3540	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3541	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3542
3543	Tokens = annotate(Code: "struct Foo {\n"
3544	" Foo() {};\n"
3545	" ~Foo() {};\n"
3546	"};");
3547	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
3548	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
3549	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);
3550	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);
3551	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3552	EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3553	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_CtorDtorDeclName);
3554	EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);
3555	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
3556	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3557	EXPECT_BRACE_KIND(Tokens[14], BK_Block);
3558
3559	Tokens = annotate(Code: "{\n"
3560	" char *a[] = {\n"
3561	" /* abc */ \"abc\",\n"
3562	"#if FOO\n"
3563	" /* xyz */ \"xyz\",\n"
3564	"#endif\n"
3565	" /* last */ \"last\"};\n"
3566	"}");
3567	ASSERT_EQ(Tokens.size(), 25u) << Tokens;
3568	EXPECT_BRACE_KIND(Tokens[0], BK_Block);
3569	EXPECT_BRACE_KIND(Tokens[7], BK_BracedInit);
3570	EXPECT_BRACE_KIND(Tokens[21], BK_BracedInit);
3571
3572	Tokens =
3573	annotate(Code: "#define SCOP_STAT(NAME, DESC) \\\n"
3574	" {\"polly\", #NAME, \"Number of rejected regions: \" DESC}");
3575	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
3576	EXPECT_BRACE_KIND(Tokens[8], BK_BracedInit);
3577	EXPECT_BRACE_KIND(Tokens[16], BK_BracedInit);
3578
3579	Tokens = annotate(Code: "struct {};");
3580	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
3581	EXPECT_BRACE_KIND(Tokens[1], BK_Block);
3582	EXPECT_BRACE_KIND(Tokens[2], BK_Block);
3583
3584	Tokens = annotate(Code: "struct : Base {};");
3585	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3586	EXPECT_BRACE_KIND(Tokens[3], BK_Block);
3587	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3588
3589	Tokens = annotate(Code: "struct Foo {};");
3590	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3591	EXPECT_BRACE_KIND(Tokens[2], BK_Block);
3592	EXPECT_BRACE_KIND(Tokens[3], BK_Block);
3593
3594	Tokens = annotate(Code: "struct ::Foo {};");
3595	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3596	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);
3597	EXPECT_BRACE_KIND(Tokens[3], BK_Block);
3598	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3599
3600	Tokens = annotate(Code: "struct NS::Foo {};");
3601	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3602	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);
3603	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_ClassHeadName);
3604	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3605	EXPECT_BRACE_KIND(Tokens[5], BK_Block);
3606
3607	Tokens = annotate(Code: "struct Foo<int> {};");
3608	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3609	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
3610	EXPECT_BRACE_KIND(Tokens[5], BK_Block);
3611	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3612
3613	Tokens = annotate(Code: "struct Foo<int>::Bar {};");
3614	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3615	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);
3616	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);
3617	EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3618	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3619
3620	Tokens = annotate(Code: "struct Foo<int> : Base {};");
3621	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3622	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
3623	EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3624	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3625
3626	Tokens = annotate(Code: "struct Foo<int> : Base::Bar {};");
3627	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
3628	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
3629	EXPECT_TOKEN(Tokens[8], tok::identifier, TT_Unknown); // Not TT_ClassHeadName.
3630	EXPECT_BRACE_KIND(Tokens[9], BK_Block);
3631	EXPECT_BRACE_KIND(Tokens[10], BK_Block);
3632
3633	Tokens = annotate(Code: "struct Foo final {};");
3634	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3635	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
3636	EXPECT_BRACE_KIND(Tokens[3], BK_Block);
3637	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3638
3639	Tokens = annotate(Code: "struct [[foo]] [[bar]] Foo final : Base1, Base2 {};");
3640	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
3641	EXPECT_TOKEN(Tokens[11], tok::identifier, TT_ClassHeadName);
3642	EXPECT_BRACE_KIND(Tokens[17], BK_Block);
3643	EXPECT_BRACE_KIND(Tokens[18], BK_Block);
3644
3645	Tokens = annotate(Code: "struct Foo x{};");
3646	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3647	EXPECT_BRACE_KIND(Tokens[3], BK_BracedInit);
3648	EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);
3649
3650	Tokens = annotate(Code: "struct ::Foo x{};");
3651	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3652	EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);
3653	EXPECT_BRACE_KIND(Tokens[5], BK_BracedInit);
3654
3655	Tokens = annotate(Code: "struct NS::Foo x{};");
3656	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3657	EXPECT_BRACE_KIND(Tokens[5], BK_BracedInit);
3658	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
3659
3660	Tokens = annotate(Code: "struct Foo<int> x{};");
3661	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3662	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
3663	EXPECT_BRACE_KIND(Tokens[7], BK_BracedInit);
3664
3665	Tokens = annotate(Code: "#ifdef DEBUG_ENABLED\n"
3666	"#else\n"
3667	"#endif\n"
3668	"class RenderingServer : Object {\n"
3669	"#ifndef DISABLE_DEPRECATED\n"
3670	" enum Features {\n"
3671	" FEATURE_SHADERS,\n"
3672	" FEATURE_MULTITHREADED,\n"
3673	" };\n"
3674	"#endif\n"
3675	"};");
3676	ASSERT_EQ(Tokens.size(), 29u) << Tokens;
3677	EXPECT_TOKEN(Tokens[8], tok::identifier, TT_ClassHeadName);
3678	EXPECT_BRACE_KIND(Tokens[11], BK_Block);
3679	EXPECT_BRACE_KIND(Tokens[17], BK_Block);
3680	EXPECT_BRACE_KIND(Tokens[22], BK_Block);
3681	EXPECT_BRACE_KIND(Tokens[26], BK_Block);
3682
3683	Tokens = annotate(Code: "{\n"
3684	"#define M(x) \\\n"
3685	" return {#x};\n"
3686	"}");
3687	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
3688	EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_BlockLBrace);
3689	EXPECT_BRACE_KIND(Tokens[0], BK_Block);
3690	EXPECT_BRACE_KIND(Tokens[8], BK_BracedInit);
3691	EXPECT_BRACE_KIND(Tokens[11], BK_BracedInit);
3692	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3693
3694	Tokens = annotate(Code: "{\n"
3695	" {\n"
3696	"#define GEN_ID(_x) char *_x{#_x}\n"
3697	" GEN_ID(one);\n"
3698	" }\n"
3699	"}");
3700	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
3701	EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_BlockLBrace);
3702	EXPECT_BRACE_KIND(Tokens[0], BK_Block);
3703	EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_BlockLBrace);
3704	EXPECT_BRACE_KIND(Tokens[1], BK_Block);
3705	EXPECT_BRACE_KIND(Tokens[11], BK_BracedInit);
3706	EXPECT_BRACE_KIND(Tokens[14], BK_BracedInit);
3707	EXPECT_BRACE_KIND(Tokens[20], BK_Block);
3708	EXPECT_BRACE_KIND(Tokens[21], BK_Block);
3709
3710	Tokens = annotate(Code: "{\n"
3711	"#define FOO \\\n"
3712	" { \\\n"
3713	" case bar: { \\\n"
3714	" break; \\\n"
3715	" } \\\n"
3716	" }\n"
3717	"}");
3718	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
3719	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_BlockLBrace);
3720	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3721	EXPECT_TOKEN(Tokens[7], tok::colon, TT_CaseLabelColon);
3722	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3723	EXPECT_BRACE_KIND(Tokens[11], BK_Block);
3724	EXPECT_BRACE_KIND(Tokens[12], BK_Block);
3725
3726	Tokens = annotate(Code: "class foo {\n"
3727	" foo() {}\n"
3728	"#if defined(_MSC_VER__clang____GNUC__FOO_) || \\\n"
3729	" (defined(__GNUC__) && defined(FOO))\n"
3730	" foo() {}\n"
3731	"#endif\n"
3732	"};");
3733	ASSERT_EQ(Tokens.size(), 36u) << Tokens;
3734	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);
3735	EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3736	EXPECT_TOKEN(Tokens[26], tok::identifier, TT_CtorDtorDeclName);
3737	EXPECT_TOKEN(Tokens[27], tok::l_paren, TT_FunctionDeclarationLParen);
3738
3739	Tokens = annotate(Code: "a = class extends goog.a {};",
3740	Style: getGoogleStyle(Language: FormatStyle::LK_JavaScript));
3741	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3742	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
3743	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_ClassLBrace);
3744	EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3745	EXPECT_TOKEN(Tokens[8], tok::r_brace, TT_ClassRBrace);
3746	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3747
3748	Tokens = annotate(Code: "a = class Foo extends goog.a {};",
3749	Style: getGoogleStyle(Language: FormatStyle::LK_JavaScript));
3750	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3751	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_ClassHeadName);
3752	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown); // Not TT_StartOfName
3753	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_ClassLBrace);
3754	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3755	EXPECT_TOKEN(Tokens[9], tok::r_brace, TT_ClassRBrace);
3756	EXPECT_BRACE_KIND(Tokens[9], BK_Block);
3757
3758	Tokens = annotate(Code: "#define FOO(X) \\\n"
3759	" struct X##_tag_ {};");
3760	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3761	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_Unknown);
3762	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_ClassHeadName);
3763	EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_StructLBrace);
3764	EXPECT_BRACE_KIND(Tokens[10], BK_Block);
3765	EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_StructRBrace);
3766	EXPECT_BRACE_KIND(Tokens[11], BK_Block);
3767
3768	Tokens = annotate(Code: "#define MACRO \\\n"
3769	" struct hash<type> { \\\n"
3770	" void f() { return; } \\\n"
3771	" };");
3772	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
3773	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_ClassHeadName);
3774	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_StructLBrace);
3775	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3776	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_FunctionDeclarationName);
3777	EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);
3778	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
3779	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3780	EXPECT_BRACE_KIND(Tokens[16], BK_Block);
3781	EXPECT_TOKEN(Tokens[17], tok::r_brace, TT_StructRBrace);
3782	EXPECT_BRACE_KIND(Tokens[17], BK_Block);
3783
3784	Tokens = annotate(Code: "#define MEMBER(NAME) NAME{\"\"}");
3785	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3786	EXPECT_BRACE_KIND(Tokens[7], BK_BracedInit);
3787	EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
3788
3789	Tokens = annotate(Code: "return lhs ^ Byte{rhs};");
3790	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3791	EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);
3792	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
3793
3794	Tokens = annotate(Code: "auto f1{&T::operator()};");
3795	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3796	EXPECT_BRACE_KIND(Tokens[2], BK_BracedInit);
3797	// Not TT_FunctionDeclarationName.
3798	EXPECT_TOKEN(Tokens[6], tok::kw_operator, TT_Unknown);
3799	EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
3800	}
3801
3802	TEST_F(TokenAnnotatorTest, UnderstandsElaboratedTypeSpecifier) {
3803	auto Tokens = annotate(Code: "auto foo() -> enum En {}");
3804	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3805	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
3806	}
3807
3808	TEST_F(TokenAnnotatorTest, BlockLBrace) {
3809	auto Tokens = annotate(Code: "{\n"
3810	" {\n"
3811	" foo();\n"
3812	" }\n"
3813	"}");
3814	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3815	EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_BlockLBrace);
3816	EXPECT_BRACE_KIND(Tokens[0], BK_Block);
3817	EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_BlockLBrace);
3818	EXPECT_BRACE_KIND(Tokens[1], BK_Block);
3819
3820	Tokens = annotate(Code: "void bar() {\n"
3821	" {\n"
3822	" foo();\n"
3823	" }\n"
3824	"}");
3825	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
3826	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
3827	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3828	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_BlockLBrace);
3829	EXPECT_BRACE_KIND(Tokens[5], BK_Block);
3830
3831	Tokens = annotate(Code: "[foo bar:{{0, 1}} baz:baz];",
3832	Style: getLLVMStyle(Language: FormatStyle::LK_ObjC));
3833	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
3834	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_Unknown); // Not TT_BlockLBrace.
3835	EXPECT_BRACE_KIND(Tokens[4], BK_Unknown); // Not BK_Block.
3836	EXPECT_BRACE_KIND(Tokens[5], BK_BracedInit);
3837	EXPECT_BRACE_KIND(Tokens[9], BK_Unknown); // Not BK_Block.
3838	EXPECT_BRACE_KIND(Tokens[10], BK_Unknown); // Not BK_Block.
3839	}
3840
3841	TEST_F(TokenAnnotatorTest, SwitchExpression) {
3842	auto Style = getLLVMStyle(Language: FormatStyle::LK_Java);
3843	auto Tokens = annotate(Code: "i = switch (day) {\n"
3844	" case THURSDAY, SATURDAY -> 8;\n"
3845	" case WEDNESDAY -> 9;\n"
3846	" default -> 1;\n"
3847	"};",
3848	Style);
3849	ASSERT_EQ(Tokens.size(), 26u) << Tokens;
3850	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_SwitchExpressionLBrace);
3851	EXPECT_TOKEN(Tokens[7], tok::kw_case, TT_SwitchExpressionLabel);
3852	EXPECT_TOKEN(Tokens[11], tok::arrow, TT_CaseLabelArrow);
3853	EXPECT_TOKEN(Tokens[14], tok::kw_case, TT_SwitchExpressionLabel);
3854	EXPECT_TOKEN(Tokens[16], tok::arrow, TT_CaseLabelArrow);
3855	EXPECT_TOKEN(Tokens[19], tok::kw_default, TT_SwitchExpressionLabel);
3856	EXPECT_TOKEN(Tokens[20], tok::arrow, TT_CaseLabelArrow);
3857	}
3858
3859	TEST_F(TokenAnnotatorTest, JavaRecord) {
3860	auto Tokens = annotate(Code: "public record MyRecord() {}",
3861	Style: getLLVMStyle(Language: FormatStyle::LK_Java));
3862	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3863	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);
3864	// Not TT_FunctionDeclarationLParen.
3865	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_Unknown);
3866	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_RecordLBrace);
3867	EXPECT_TOKEN(Tokens[6], tok::r_brace, TT_RecordRBrace);
3868	}
3869
3870	TEST_F(TokenAnnotatorTest, CppAltOperatorKeywords) {
3871	auto Tokens = annotate(Code: "a = b and c;");
3872	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3873	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator);
3874
3875	Tokens = annotate(Code: "a = b and_eq c;");
3876	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3877	EXPECT_TOKEN(Tokens[3], tok::ampequal, TT_BinaryOperator);
3878
3879	Tokens = annotate(Code: "a = b bitand c;");
3880	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3881	EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
3882
3883	Tokens = annotate(Code: "a = b bitor c;");
3884	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3885	EXPECT_TOKEN(Tokens[3], tok::pipe, TT_BinaryOperator);
3886
3887	Tokens = annotate(Code: "a = b compl c;");
3888	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3889	EXPECT_TOKEN(Tokens[3], tok::tilde, TT_UnaryOperator);
3890
3891	Tokens = annotate(Code: "a = b not c;");
3892	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3893	EXPECT_TOKEN(Tokens[3], tok::exclaim, TT_UnaryOperator);
3894
3895	Tokens = annotate(Code: "a = b not_eq c;");
3896	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3897	EXPECT_TOKEN(Tokens[3], tok::exclaimequal, TT_BinaryOperator);
3898
3899	Tokens = annotate(Code: "a = b or c;");
3900	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3901	EXPECT_TOKEN(Tokens[3], tok::pipepipe, TT_BinaryOperator);
3902
3903	Tokens = annotate(Code: "return segment < *this or *this < segment;");
3904	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3905	EXPECT_TOKEN(Tokens[5], tok::pipepipe, TT_BinaryOperator);
3906	EXPECT_TOKEN(Tokens[6], tok::star, TT_UnaryOperator);
3907
3908	Tokens = annotate(Code: "a = b or_eq c;");
3909	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3910	EXPECT_TOKEN(Tokens[3], tok::pipeequal, TT_BinaryOperator);
3911
3912	Tokens = annotate(Code: "a = b xor c;");
3913	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3914	EXPECT_TOKEN(Tokens[3], tok::caret, TT_BinaryOperator);
3915
3916	Tokens = annotate(Code: "a = b xor_eq c;");
3917	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3918	EXPECT_TOKEN(Tokens[3], tok::caretequal, TT_BinaryOperator);
3919
3920	const auto StyleC = getLLVMStyle(Language: FormatStyle::LK_C);
3921
3922	Tokens = annotate(Code: "xor = foo;", Style: StyleC);
3923	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
3924	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
3925
3926	Tokens = annotate(Code: "int xor = foo;", Style: StyleC);
3927	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3928	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
3929	}
3930
3931	TEST_F(TokenAnnotatorTest, CppOnlyKeywordInC) {
3932	auto Tokens = annotate(Code: "int maximized = new & STATE_MAXIMIZED;",
3933	Style: getLLVMStyle(Language: FormatStyle::LK_C));
3934	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3935	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown); // Not tok::kw_new
3936	EXPECT_TOKEN(Tokens[4], tok::amp, TT_BinaryOperator);
3937	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown); // Not TT_StartOfName
3938	}
3939
3940	TEST_F(TokenAnnotatorTest, FunctionTryBlock) {
3941	auto Tokens =
3942	annotate(Code: "Foo::Foo(int x, int y) try\n"
3943	" : foo{[] -> std::string { return {}; }(), x}, bar{y} {\n"
3944	"} catch (...) {\n"
3945	"}");
3946	ASSERT_EQ(Tokens.size(), 45u) << Tokens;
3947	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_CtorDtorDeclName);
3948	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);
3949	EXPECT_TOKEN(Tokens[11], tok::colon, TT_CtorInitializerColon);
3950	EXPECT_TOKEN(Tokens[14], tok::l_square, TT_LambdaLSquare);
3951	EXPECT_TOKEN(Tokens[16], tok::arrow, TT_LambdaArrow);
3952	EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
3953	EXPECT_TOKEN(Tokens[31], tok::comma, TT_CtorInitializerComma);
3954	EXPECT_TOKEN(Tokens[36], tok::l_brace, TT_FunctionLBrace);
3955	}
3956
3957	TEST_F(TokenAnnotatorTest, TypenameMacro) {
3958	auto Style = getLLVMStyle();
3959	Style.TypenameMacros.push_back(x: "STRUCT");
3960
3961	auto Tokens = annotate(Code: "STRUCT(T, B) { int i; };", Style);
3962	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
3963	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TypenameMacro);
3964	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_TypeDeclarationParen);
3965	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_TypeDeclarationParen);
3966	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_Unknown);
3967	}
3968
3969	TEST_F(TokenAnnotatorTest, GNULanguageStandard) {
3970	auto Style = getGNUStyle();
3971	EXPECT_EQ(Style.Standard, FormatStyle::LS_Latest);
3972
3973	auto Tokens = annotate(Code: "return 1 <=> 2;", Style);
3974	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3975	EXPECT_TOKEN(Tokens[2], tok::spaceship, TT_BinaryOperator);
3976	}
3977
3978	TEST_F(TokenAnnotatorTest, SplitPenalty) {
3979	auto Style = getLLVMStyle();
3980	Style.ColumnLimit = 20;
3981
3982	auto Tokens = annotate(Code: "class foo {\n"
3983	" auto bar()\n"
3984	" -> bool;\n"
3985	"};",
3986	Style);
3987	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
3988	EXPECT_TOKEN(Tokens[7], tok::arrow, TT_TrailingReturnArrow);
3989	EXPECT_SPLIT_PENALTY(Tokens[7], 23u);
3990	}
3991
3992	TEST_F(TokenAnnotatorTest, TemplateName) {
3993	constexpr StringRef Code{"return Foo < A || B > (C ^ D);"};
3994
3995	auto Tokens = annotate(Code);
3996	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3997	EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
3998	EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);
3999
4000	auto Style = getLLVMStyle();
4001	Style.TemplateNames.push_back(x: "Foo");
4002
4003	Tokens = annotate(Code, Style);
4004	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_TemplateName);
4005	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
4006	EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
4007	}
4008
4009	TEST_F(TokenAnnotatorTest, TemplateInstantiation) {
4010	auto Tokens = annotate(Code: "return FixedInt<N | M>();");
4011	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
4012	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
4013	EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
4014
4015	Tokens = annotate(Code: "return FixedInt<N | M>(foo);");
4016	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
4017	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
4018	EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
4019
4020	Tokens = annotate(Code: "return std::conditional_t<T::value == U::value, T, U>{};");
4021	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
4022	EXPECT_TOKEN(Tokens[4], tok::less, TT_TemplateOpener);
4023	EXPECT_TOKEN(Tokens[16], tok::greater, TT_TemplateCloser);
4024
4025	Tokens =
4026	annotate(Code: "auto x{std::conditional_t<T::value == U::value, T, U>{}};");
4027	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
4028	EXPECT_TOKEN(Tokens[6], tok::less, TT_TemplateOpener);
4029	EXPECT_TOKEN(Tokens[18], tok::greater, TT_TemplateCloser);
4030
4031	Tokens = annotate(
4032	Code: "std::uint16_t kMTU = std::conditional<\n"
4033	" kTypeKind == KindA,\n"
4034	" std::integral_constant<std::uint16_t, kIoSockMtu>>::type::value;");
4035	ASSERT_EQ(Tokens.size(), 30u) << Tokens;
4036	EXPECT_TOKEN(Tokens[8], tok::less, TT_TemplateOpener);
4037	EXPECT_TOKEN(Tokens[16], tok::less, TT_TemplateOpener);
4038	EXPECT_TOKEN(Tokens[22], tok::greater, TT_TemplateCloser);
4039	EXPECT_TOKEN(Tokens[23], tok::greater, TT_TemplateCloser);
4040	}
4041
4042	TEST_F(TokenAnnotatorTest, VariableTemplate) {
4043	auto Style = getLLVMStyle();
4044	Style.VariableTemplates.push_back(x: "a");
4045
4046	auto Tokens = annotate(Code: "auto t3 = (a<int>) + b;", Style);
4047	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
4048	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_VariableTemplate);
4049	EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);
4050	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
4051	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown); // Not TT_CastRParen
4052	EXPECT_TOKEN(Tokens[9], tok::plus, TT_BinaryOperator);
4053	}
4054
4055	TEST_F(TokenAnnotatorTest, SwitchInMacroArgument) {
4056	auto Tokens = annotate(Code: "FOOBAR(switch);\n"
4057	"void f() {}");
4058	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
4059	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_FunctionLBrace);
4060	}
4061
4062	TEST_F(TokenAnnotatorTest, AfterPPDirective) {
4063	auto Tokens = annotate(Code: "#error -- My error message");
4064
4065	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
4066	EXPECT_TOKEN(Tokens[2], tok::minusminus, TT_AfterPPDirective);
4067	}
4068
4069	TEST_F(TokenAnnotatorTest, UserDefinedConversionFunction) {
4070	auto Tokens = annotate(Code: "operator int(void);");
4071	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
4072	EXPECT_TOKEN(Tokens[0], tok::kw_operator, TT_FunctionDeclarationName);
4073	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
4074
4075	Tokens = annotate(Code: "explicit operator int *();");
4076	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
4077	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
4078	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
4079	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);
4080
4081	Tokens = annotate(Code: "operator int &();");
4082	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
4083	EXPECT_TOKEN(Tokens[0], tok::kw_operator, TT_FunctionDeclarationName);
4084	EXPECT_TOKEN(Tokens[2], tok::amp, TT_PointerOrReference);
4085	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);
4086
4087	Tokens = annotate(Code: "operator auto() const { return 2; }");
4088	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
4089	EXPECT_TOKEN(Tokens[0], tok::kw_operator, TT_FunctionDeclarationName);
4090	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
4091	EXPECT_TOKEN(Tokens[4], tok::kw_const, TT_TrailingAnnotation);
4092	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);
4093
4094	Tokens = annotate(Code: "operator decltype(auto)() const;");
4095	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
4096	EXPECT_TOKEN(Tokens[0], tok::kw_operator, TT_FunctionDeclarationName);
4097	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_TypeDeclarationParen);
4098	EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_TypeDeclarationParen);
4099	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_FunctionDeclarationLParen);
4100	EXPECT_TOKEN(Tokens[7], tok::kw_const, TT_TrailingAnnotation);
4101
4102	Tokens = annotate(Code: "virtual operator Foo() = 0;");
4103	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
4104	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
4105	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);
4106
4107	Tokens = annotate(Code: "operator Foo() override { return Foo(); }");
4108	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
4109	EXPECT_TOKEN(Tokens[0], tok::kw_operator, TT_FunctionDeclarationName);
4110	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
4111	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);
4112
4113	Tokens = annotate(Code: "friend Bar::operator Foo();");
4114	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
4115	EXPECT_TOKEN(Tokens[3], tok::kw_operator, TT_FunctionDeclarationName);
4116	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_FunctionDeclarationLParen);
4117	}
4118
4119	TEST_F(TokenAnnotatorTest, UTF8StringLiteral) {
4120	auto Tokens = annotate(Code: "return u8\"foo\";", Style: getLLVMStyle(Language: FormatStyle::LK_C));
4121	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
4122	EXPECT_TOKEN(Tokens[1], tok::utf8_string_literal, TT_Unknown);
4123	}
4124
4125	TEST_F(TokenAnnotatorTest, C23DigitSeparator) {
4126	auto Tokens = annotate(Code: "return 1'000;", Style: getLLVMStyle(Language: FormatStyle::LK_C));
4127	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
4128	EXPECT_EQ(Tokens[1]->TokenText, "1'000");
4129	EXPECT_TOKEN(Tokens[2], tok::semi, TT_Unknown);
4130	}
4131
4132	TEST_F(TokenAnnotatorTest, IdentifierPackage) {
4133	auto Tokens = annotate(Code: "auto package;");
4134	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
4135	EXPECT_FALSE(Tokens[0]->isObjCAccessSpecifier());
4136	}
4137
4138	TEST_F(TokenAnnotatorTest, UserDefinedLiteral) {
4139	auto Tokens = annotate(Code: "auto dollars = 2_$;");
4140	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
4141	EXPECT_EQ(Tokens[3]->TokenText, "2_$");
4142	}
4143
4144	TEST_F(TokenAnnotatorTest, EnumColonInTypedef) {
4145	auto Tokens = annotate(Code: "typedef enum : int {} foo;");
4146	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
4147	EXPECT_TOKEN(Tokens[2], tok::colon, TT_Unknown); // Not TT_InheritanceColon.
4148	}
4149
4150	TEST_F(TokenAnnotatorTest, BitFieldColon) {
4151	auto Tokens = annotate(Code: "class C {\n"
4152	" int f : SIZE;\n"
4153	"};");
4154	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
4155	EXPECT_TOKEN(Tokens[5], tok::colon, TT_BitFieldColon);
4156	}
4157
4158	TEST_F(TokenAnnotatorTest, JsonCodeInRawString) {
4159	auto Tokens = annotate(Code: "{\n"
4160	" \"foo\": \"bar\",\n"
4161	" \"str\": \"test\"\n"
4162	"}",
4163	Style: getLLVMStyle(Language: FormatStyle::LK_Json));
4164	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
4165	EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_DictLiteral);
4166	EXPECT_TOKEN(Tokens[1], tok::string_literal, TT_SelectorName);
4167	EXPECT_TOKEN(Tokens[2], tok::colon, TT_DictLiteral);
4168	EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_SelectorName);
4169	EXPECT_TOKEN(Tokens[6], tok::colon, TT_DictLiteral);
4170	}
4171
4172	TEST_F(TokenAnnotatorTest, LineCommentTrailingBackslash) {
4173	auto Tokens = annotate(Code: "// a \\\n"
4174	"// b");
4175	ASSERT_EQ(Tokens.size(), 3u) << Tokens;
4176	EXPECT_TOKEN(Tokens[1], tok::comment, TT_LineComment);
4177	}
4178
4179	TEST_F(TokenAnnotatorTest, ArrowAfterSubscript) {
4180	auto Tokens =
4181	annotate(Code: "return (getStructType()->getElements())[eIdx]->getName();");
4182	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
4183	// Not TT_LambdaArrow.
4184	EXPECT_TOKEN(Tokens[13], tok::arrow, TT_Unknown);
4185	}
4186
4187	} // namespace
4188	} // namespace format
4189	} // namespace clang
4190