1	//===- unittest/Format/TokenAnnotatorTest.cpp - Formatting unit tests -----===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "clang/Format/Format.h"
10
11	#include "FormatTestUtils.h"
12	#include "TestLexer.h"
13	#include "gtest/gtest.h"
14
15	namespace clang {
16	namespace format {
17
18	// Not really the equality, but everything we need.
19	static bool operator==(const FormatToken &LHS,
20	const FormatToken &RHS) noexcept {
21	return LHS.Tok.getKind() == RHS.Tok.getKind() &&
22	LHS.getType() == RHS.getType();
23	}
24
25	namespace {
26
27	class TokenAnnotatorTest : public testing::Test {
28	protected:
29	TokenList annotate(StringRef Code,
30	const FormatStyle &Style = getLLVMStyle()) {
31	return TestLexer(Allocator, Buffers, Style).annotate(Code);
32	}
33	llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
34	std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
35	};
36
37	#define EXPECT_TOKEN_KIND(FormatTok, Kind) \
38	EXPECT_EQ((FormatTok)->Tok.getKind(), Kind) << *(FormatTok)
39	#define EXPECT_TOKEN_TYPE(FormatTok, Type) \
40	EXPECT_EQ((FormatTok)->getType(), Type) << *(FormatTok)
41	#define EXPECT_TOKEN_PRECEDENCE(FormatTok, Prec) \
42	EXPECT_EQ((FormatTok)->getPrecedence(), Prec) << *(FormatTok)
43	#define EXPECT_BRACE_KIND(FormatTok, Kind) \
44	EXPECT_EQ(FormatTok->getBlockKind(), Kind) << *(FormatTok)
45	#define EXPECT_SPLIT_PENALTY(FormatTok, Penalty) \
46	EXPECT_EQ(FormatTok->SplitPenalty, Penalty) << *(FormatTok)
47	#define EXPECT_TOKEN(FormatTok, Kind, Type) \
48	do { \
49	EXPECT_TOKEN_KIND(FormatTok, Kind); \
50	EXPECT_TOKEN_TYPE(FormatTok, Type); \
51	} while (false)
52
53	TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) {
54	auto Tokens = annotate(Code: "auto x = [](const decltype(x) &ptr) {};");
55	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
56	EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);
57	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);
58	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);
59	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
60	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
61
62	Tokens = annotate(Code: "auto x = [](const decltype(x) *ptr) {};");
63	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
64	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
65	EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
66
67	Tokens = annotate(Code: "#define lambda [](const decltype(x) &ptr) {}");
68	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
69	EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);
70	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);
71	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);
72	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
73	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
74
75	Tokens = annotate(Code: "#define lambda [](const decltype(x) *ptr) {}");
76	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
77	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
78	EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
79
80	Tokens = annotate(Code: "void f() {\n"
81	" while (p < a && *p == 'a')\n"
82	" p++;\n"
83	"}");
84	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
85	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
86	EXPECT_TOKEN(Tokens[11], tok::star, TT_UnaryOperator);
87
88	Tokens = annotate(Code: "case *x:");
89	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
90	EXPECT_TOKEN(Tokens[1], tok::star, TT_UnaryOperator);
91	Tokens = annotate(Code: "case &x:");
92	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
93	EXPECT_TOKEN(Tokens[1], tok::amp, TT_UnaryOperator);
94
95	Tokens = annotate(Code: "bool b = 3 == int{3} && true;");
96	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
97	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
98
99	Tokens = annotate(Code: "struct {\n"
100	"} *ptr;");
101	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
102	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
103	Tokens = annotate(Code: "union {\n"
104	"} *ptr;");
105	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
106	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
107	Tokens = annotate(Code: "class {\n"
108	"} *ptr;");
109	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
110	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
111
112	Tokens = annotate(Code: "struct {\n"
113	"} &&ptr = {};");
114	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
115	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
116	Tokens = annotate(Code: "union {\n"
117	"} &&ptr = {};");
118	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
119	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
120	Tokens = annotate(Code: "class {\n"
121	"} &&ptr = {};");
122	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
123	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
124	Tokens = annotate(Code: "int i = int{42} * 2;");
125	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
126	EXPECT_TOKEN(Tokens[7], tok::star, TT_BinaryOperator);
127
128	Tokens = annotate(Code: "delete[] *ptr;");
129	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
130	EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
131	Tokens = annotate(Code: "delete[] **ptr;");
132	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
133	EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
134	EXPECT_TOKEN(Tokens[4], tok::star, TT_UnaryOperator);
135	Tokens = annotate(Code: "delete[] *(ptr);");
136	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
137	EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
138
139	Tokens = annotate(Code: "void f() { void (*fnptr)(char* foo); }");
140	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
141	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
142	// FIXME: The star of a function pointer probably makes more sense as
143	// TT_PointerOrReference.
144	EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
145	EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
146
147	Tokens = annotate(Code: "void f() { void (*fnptr)(t* foo); }");
148	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
149	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
150	EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
151	EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
152
153	Tokens = annotate(Code: "int f3() { return sizeof(Foo&); }");
154	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
155	EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
156
157	Tokens = annotate(Code: "int f4() { return sizeof(Foo&&); }");
158	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
159	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
160
161	Tokens = annotate(Code: "void f5() { int f6(Foo&, Bar&); }");
162	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
163	EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
164	EXPECT_TOKEN(Tokens[12], tok::amp, TT_PointerOrReference);
165
166	Tokens = annotate(Code: "void f7() { int f8(Foo&&, Bar&&); }");
167	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
168	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
169	EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_PointerOrReference);
170
171	Tokens = annotate(Code: "Type1 &val1 = val2;");
172	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
173	EXPECT_TOKEN(Tokens[1], tok::amp, TT_PointerOrReference);
174
175	Tokens = annotate(Code: "Type1 *val1 = &val2;");
176	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
177	EXPECT_TOKEN(Tokens[1], tok::star, TT_PointerOrReference);
178	EXPECT_TOKEN(Tokens[4], tok::amp, TT_UnaryOperator);
179
180	Tokens = annotate(Code: "val1 & val2;");
181	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
182	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
183
184	Tokens = annotate(Code: "val1 & val2.member;");
185	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
186	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
187
188	Tokens = annotate(Code: "val1 & val2.*member;");
189	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
190	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
191
192	Tokens = annotate(Code: "val1.*member & val2;");
193	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
194	EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
195
196	Tokens = annotate(Code: "val1 & val2->*member;");
197	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
198	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
199
200	Tokens = annotate(Code: "val1->member & val2;");
201	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
202	EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
203
204	Tokens = annotate(Code: "val1 & val2 & val3;");
205	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
206	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
207	EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
208
209	Tokens = annotate(Code: "val1 & val2 // comment\n"
210	" & val3;");
211	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
212	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
213	EXPECT_TOKEN(Tokens[4], tok::amp, TT_BinaryOperator);
214
215	Tokens =
216	annotate(Code: "val1 & val2.member & val3.member() & val4 & val5->member;");
217	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
218	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
219	EXPECT_TOKEN(Tokens[5], tok::amp, TT_BinaryOperator);
220	EXPECT_TOKEN(Tokens[11], tok::amp, TT_BinaryOperator);
221	EXPECT_TOKEN(Tokens[13], tok::amp, TT_BinaryOperator);
222
223	Tokens = annotate(Code: "class c {\n"
224	" void func(type &a) { a & member; }\n"
225	" anotherType &member;\n"
226	"}");
227	ASSERT_EQ(Tokens.size(), 22u) << Tokens;
228	EXPECT_TOKEN(Tokens[7], tok::amp, TT_PointerOrReference);
229	EXPECT_TOKEN(Tokens[12], tok::amp, TT_BinaryOperator);
230	EXPECT_TOKEN(Tokens[17], tok::amp, TT_PointerOrReference);
231
232	Tokens = annotate(Code: "struct S {\n"
233	" auto Mem = C & D;\n"
234	"}");
235	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
236	EXPECT_TOKEN(Tokens[7], tok::amp, TT_BinaryOperator);
237
238	Tokens =
239	annotate(Code: "template <typename T> void swap() noexcept(Bar<T> && Foo<T>);");
240	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
241	EXPECT_TOKEN(Tokens[15], tok::ampamp, TT_BinaryOperator);
242
243	Tokens = annotate(Code: "template <typename T> struct S {\n"
244	" explicit(Bar<T> && Foo<T>) S(const S &);\n"
245	"};");
246	ASSERT_EQ(Tokens.size(), 30u) << Tokens;
247	EXPECT_TOKEN(Tokens[14], tok::ampamp, TT_BinaryOperator);
248
249	Tokens = annotate(Code: "template <bool B = C && D> struct S {};");
250	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
251	EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_BinaryOperator);
252
253	Tokens = annotate(Code: "template <typename T, bool B = C && D> struct S {};");
254	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
255	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
256
257	Tokens = annotate(Code: "template <typename T, typename U = T&&> struct S {};");
258	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
259	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
260
261	Tokens = annotate(Code: "template <typename T = int (*)(int)> struct S {};");
262	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
263	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
264	EXPECT_TOKEN(Tokens[7], tok::star, TT_PointerOrReference);
265
266	Tokens = annotate(Code: "Foo<A && B> a = {};");
267	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
268	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator);
269
270	Tokens = annotate(Code: "Foo<A &&> a = {};");
271	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
272	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
273
274	Tokens = annotate(Code: "template <typename T>\n"
275	"enable_if_t<is_integral_v<T>, bool> // comment\n"
276	"operator~(T &a);");
277	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
278	EXPECT_TOKEN(Tokens[19], tok::amp, TT_PointerOrReference);
279
280	Tokens = annotate(Code: "template <enable_if_t<foo && !bar>* = nullptr> void f();");
281	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
282	EXPECT_TOKEN(Tokens[5], tok::ampamp, TT_BinaryOperator);
283
284	Tokens =
285	annotate(Code: "auto foo() noexcept(noexcept(bar()) && "
286	"trait<std::decay_t<decltype(bar())>> && noexcept(baz())) {}");
287	ASSERT_EQ(Tokens.size(), 38u) << Tokens;
288	EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_BinaryOperator);
289	EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator);
290
291	Tokens = annotate(Code: "foo = *i < *j && *j > *k;");
292	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
293	EXPECT_TOKEN(Tokens[4], tok::less, TT_BinaryOperator);
294	EXPECT_TOKEN(Tokens[7], tok::ampamp, TT_BinaryOperator);
295	EXPECT_TOKEN(Tokens[10], tok::greater, TT_BinaryOperator);
296
297	Tokens = annotate(Code: "if (Foo *foo; bar)");
298	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
299	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
300
301	Tokens = annotate(Code: "if (Foo **foo(); bar)");
302	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
303	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
304	EXPECT_TOKEN(Tokens[4], tok::star, TT_PointerOrReference);
305
306	Tokens = annotate(Code: "if (Foo *&foo{a}; bar)");
307	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
308	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
309	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
310
311	FormatStyle Style = getLLVMStyle();
312	Style.TypeNames.push_back(x: "MYI");
313	Tokens = annotate(Code: "if (MYI *p{nullptr})", Style);
314	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
315	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TypeName);
316	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
317
318	Style.TypeNames.push_back(x: "Class");
319	Tokens = annotate(Code: "if (Class *obj {getObj()})", Style);
320	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
321	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TypeName);
322	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
323
324	Tokens = annotate(Code: "class Foo {\n"
325	" void operator<() {}\n"
326	" Foo &f;\n"
327	"};");
328	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
329	EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);
330	EXPECT_TOKEN(Tokens[5], tok::less, TT_OverloadedOperator);
331	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen);
332	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);
333	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
334
335	Tokens = annotate(Code: "if (new && num) {\n"
336	" new = 1;\n"
337	"}\n"
338	"if (!delete && num) {\n"
339	" delete = 1;\n"
340	"}");
341	ASSERT_EQ(Tokens.size(), 26u) << Tokens;
342	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator);
343	EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator);
344
345	Tokens = annotate(Code: "#define FOO \\\n"
346	" void foo() { f(a * b); }");
347	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
348	EXPECT_TOKEN(Tokens[11], tok::star, TT_BinaryOperator);
349
350	Tokens = annotate(Code: "for (int i; Foo *&foo : foos)");
351	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
352	EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);
353	EXPECT_TOKEN(Tokens[7], tok::amp, TT_PointerOrReference);
354	EXPECT_TOKEN(Tokens[9], tok::colon, TT_RangeBasedForLoopColon);
355
356	Tokens = annotate(Code: "#define FOO auto Foo = [] { f(a * b); };");
357	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
358	EXPECT_TOKEN(Tokens[12], tok::star, TT_BinaryOperator);
359
360	Tokens = annotate(Code: "namespace {\n"
361	"#define FOO(x) void foo(a##x *b);\n"
362	"}");
363	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
364	EXPECT_TOKEN(Tokens[14], tok::star, TT_PointerOrReference);
365
366	Tokens = annotate(Code: "#define foo(x) _Generic(x, bar *: 1, default: 0)");
367	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
368	EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
369
370	Tokens = annotate(Code: "Thingy kConfig = {\n"
371	" 1,\n"
372	" (uint16_t)(kScale * height_pixels),\n"
373	"};");
374	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
375	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_CastRParen);
376	EXPECT_TOKEN(Tokens[11], tok::star, TT_BinaryOperator);
377
378	Tokens = annotate(Code: "template <typename T>\n"
379	"concept C = requires(T a, T b) { a && b; };");
380	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
381	EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_RequiresExpressionLBrace);
382	EXPECT_TOKEN(Tokens[18], tok::ampamp, TT_BinaryOperator);
383
384	Tokens = annotate(Code: "template <typename T, typename V>\n"
385	"concept CheckMultiplicableBy = requires(T a, V b) {\n"
386	" { a * b } -> std::same_as<T>;\n"
387	"};");
388	ASSERT_EQ(Tokens.size(), 36u) << Tokens;
389	EXPECT_TOKEN(Tokens[19], tok::l_brace, TT_RequiresExpressionLBrace);
390	EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_CompoundRequirementLBrace);
391	EXPECT_TOKEN(Tokens[22], tok::star, TT_BinaryOperator);
392
393	Tokens = annotate(Code: "return s.operator int *();");
394	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
395	// Not TT_FunctionDeclarationName.
396	EXPECT_TOKEN(Tokens[3], tok::kw_operator, TT_Unknown);
397	EXPECT_TOKEN(Tokens[5], tok::star, TT_PointerOrReference);
398	// Not TT_FunctionDeclarationLParen.
399	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_Unknown);
400
401	Tokens = annotate(Code: "B &b = x.operator B &();");
402	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
403	EXPECT_TOKEN(Tokens[8], tok::amp, TT_PointerOrReference);
404
405	Tokens = annotate(Code: "int8_t *a = MacroCall(int8_t, width * height * length);");
406	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
407	EXPECT_TOKEN(Tokens[9], tok::star, TT_BinaryOperator);
408	EXPECT_TOKEN(Tokens[11], tok::star, TT_BinaryOperator);
409
410	Tokens = annotate(Code: "float foo[3] = {M * H * H, H * M * H, H * H * M};");
411	ASSERT_EQ(Tokens.size(), 27u) << Tokens;
412	EXPECT_TOKEN(Tokens[16], tok::star, TT_BinaryOperator);
413	EXPECT_TOKEN(Tokens[22], tok::star, TT_BinaryOperator);
414
415	Tokens = annotate(Code: "NSError *__autoreleasing *foo;",
416	Style: getLLVMStyle(Language: FormatStyle::LK_ObjC));
417	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
418	EXPECT_TOKEN(Tokens[1], tok::star, TT_PointerOrReference);
419	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
420	}
421
422	TEST_F(TokenAnnotatorTest, UnderstandsUsesOfPlusAndMinus) {
423	auto Tokens = annotate(Code: "x - 0");
424	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
425	EXPECT_TOKEN(Tokens[1], tok::minus, TT_BinaryOperator);
426	Tokens = annotate(Code: "0 + 0");
427	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
428	EXPECT_TOKEN(Tokens[1], tok::plus, TT_BinaryOperator);
429	Tokens = annotate(Code: "x + +0");
430	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
431	EXPECT_TOKEN(Tokens[2], tok::plus, TT_UnaryOperator);
432	Tokens = annotate(Code: "x ? -0 : +0");
433	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
434	EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
435	EXPECT_TOKEN(Tokens[5], tok::plus, TT_UnaryOperator);
436	Tokens = annotate(Code: "(-0)");
437	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
438	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
439	Tokens = annotate(Code: "0, -0");
440	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
441	EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
442	Tokens = annotate(Code: "for (; -1;) {\n}");
443	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
444	EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
445	Tokens = annotate(Code: "x = -1;");
446	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
447	EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
448	Tokens = annotate(Code: "x[-1]");
449	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
450	EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
451	Tokens = annotate(Code: "x = {-1};");
452	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
453	EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
454	Tokens = annotate(Code: "case -x:");
455	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
456	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
457	Tokens = annotate(Code: "co_await -x;");
458	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
459	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
460	Tokens = annotate(Code: "co_return -x;");
461	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
462	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
463	Tokens = annotate(Code: "co_yield -x;");
464	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
465	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
466	Tokens = annotate(Code: "delete -x;");
467	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
468	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
469	Tokens = annotate(Code: "return -x;");
470	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
471	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
472	Tokens = annotate(Code: "throw -x;");
473	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
474	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
475	Tokens = annotate(Code: "sizeof -x");
476	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
477	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
478	Tokens = annotate(Code: "co_await +x;");
479	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
480	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
481	Tokens = annotate(Code: "co_return +x;");
482	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
483	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
484	Tokens = annotate(Code: "co_yield +x;");
485	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
486	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
487	Tokens = annotate(Code: "delete +x;");
488	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
489	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
490	Tokens = annotate(Code: "return +x;");
491	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
492	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
493	Tokens = annotate(Code: "throw +x;");
494	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
495	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
496	Tokens = annotate(Code: "sizeof +x");
497	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
498	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
499	Tokens = annotate(Code: "(int)-x");
500	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
501	EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
502	Tokens = annotate(Code: "(-x)");
503	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
504	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
505	Tokens = annotate(Code: "!+x");
506	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
507	EXPECT_TOKEN(Tokens[0], tok::exclaim, TT_UnaryOperator);
508	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
509	}
510
511	TEST_F(TokenAnnotatorTest, UnderstandsClasses) {
512	auto Tokens = annotate(Code: "class C {};");
513	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
514	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
515	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);
516	EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_ClassRBrace);
517
518	Tokens = annotate(Code: "const class C {} c;");
519	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
520	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);
521	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_ClassLBrace);
522	EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_ClassRBrace);
523
524	Tokens = annotate(Code: "const class {} c;");
525	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
526	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);
527	EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_ClassRBrace);
528
529	Tokens = annotate(Code: "class [[deprecated(\"\")]] C { int i; };");
530	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
531	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_ClassHeadName);
532	EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_ClassLBrace);
533	EXPECT_TOKEN(Tokens[14], tok::r_brace, TT_ClassRBrace);
534	}
535
536	TEST_F(TokenAnnotatorTest, UnderstandsStructs) {
537	auto Tokens = annotate(Code: "struct S {};");
538	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
539	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
540	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_StructLBrace);
541	EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_StructRBrace);
542
543	Tokens = annotate(Code: "struct macro(a) S {};");
544	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
545	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_ClassHeadName);
546	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_StructLBrace);
547	EXPECT_TOKEN(Tokens[7], tok::r_brace, TT_StructRBrace);
548
549	Tokens = annotate(Code: "struct EXPORT_MACRO [[nodiscard]] C { int i; };");
550	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
551	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_ClassHeadName);
552	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_StructLBrace);
553	EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_StructRBrace);
554
555	Tokens = annotate(Code: "struct [[deprecated]] [[nodiscard]] C { int i; };");
556	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
557	EXPECT_TOKEN(Tokens[11], tok::identifier, TT_ClassHeadName);
558	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_StructLBrace);
559	EXPECT_TOKEN(Tokens[16], tok::r_brace, TT_StructRBrace);
560
561	Tokens = annotate(Code: "struct macro(a) S {\n"
562	" void f(T &t);\n"
563	"};");
564	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
565	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_ClassHeadName);
566	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_StructLBrace);
567	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
568	EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_StructRBrace);
569
570	Tokens = annotate(Code: "template <typename T> struct S<const T[N]> {};");
571	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
572	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);
573	EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);
574	EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);
575	EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
576	EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_StructLBrace);
577	EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_StructRBrace);
578
579	Tokens = annotate(Code: "template <typename T> struct S<T const[N]> {};");
580	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
581	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);
582	EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);
583	EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);
584	EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
585	EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_StructLBrace);
586	EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_StructRBrace);
587
588	Tokens = annotate(Code: "template <typename T, unsigned n> struct S<T const[n]> {\n"
589	" void f(T const (&a)[n]);\n"
590	"};");
591	ASSERT_EQ(Tokens.size(), 35u) << Tokens;
592	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_ClassHeadName);
593	EXPECT_TOKEN(Tokens[10], tok::less, TT_TemplateOpener);
594	EXPECT_TOKEN(Tokens[13], tok::l_square, TT_ArraySubscriptLSquare);
595	EXPECT_TOKEN(Tokens[16], tok::greater, TT_TemplateCloser);
596	EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_StructLBrace);
597	EXPECT_TOKEN(Tokens[23], tok::l_paren, TT_FunctionTypeLParen);
598	EXPECT_TOKEN(Tokens[24], tok::amp, TT_UnaryOperator);
599	EXPECT_TOKEN(Tokens[27], tok::l_square, TT_ArraySubscriptLSquare);
600	EXPECT_TOKEN(Tokens[32], tok::r_brace, TT_StructRBrace);
601
602	Tokens = annotate(Code: "template <typename T, enum E e> struct S {};");
603	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
604	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_ClassHeadName);
605	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_StructLBrace);
606
607	Tokens = annotate(
608	Code: "template <> struct __declspec(foo) Op<Bar *> : OpImpl<Bar *> {};");
609	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
610	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_AttributeLParen);
611	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_AttributeRParen);
612	EXPECT_TOKEN(Tokens[8], tok::identifier, TT_ClassHeadName);
613	EXPECT_TOKEN(Tokens[13], tok::colon, TT_InheritanceColon);
614	EXPECT_TOKEN(Tokens[19], tok::l_brace, TT_StructLBrace);
615	EXPECT_TOKEN(Tokens[20], tok::r_brace, TT_StructRBrace);
616
617	constexpr StringRef Code{"struct EXPORT StructName {};"};
618
619	Tokens = annotate(Code);
620	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
621	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);
622	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_StructLBrace);
623	EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_StructRBrace);
624
625	auto Style = getLLVMStyle();
626	Style.AttributeMacros.push_back(x: "EXPORT");
627	Tokens = annotate(Code, Style);
628	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
629	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_AttributeMacro);
630	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);
631	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_StructLBrace);
632	EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_StructRBrace);
633	}
634
635	TEST_F(TokenAnnotatorTest, UnderstandsUnions) {
636	auto Tokens = annotate(Code: "union U {};");
637	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
638	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
639	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
640	EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_UnionRBrace);
641
642	Tokens = annotate(Code: "union U { void f() { return; } };");
643	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
644	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
645	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
646	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
647	EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_UnionRBrace);
648	}
649
650	TEST_F(TokenAnnotatorTest, UnderstandsEnums) {
651	auto Tokens = annotate(Code: "enum E {};");
652	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
653	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_EnumLBrace);
654	EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_EnumRBrace);
655	}
656
657	TEST_F(TokenAnnotatorTest, UnderstandsDefaultedAndDeletedFunctions) {
658	auto Tokens = annotate(Code: "auto operator<=>(const T &) const & = default;");
659	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
660	EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
661
662	Tokens = annotate(Code: "template <typename T> void F(T) && = delete;");
663	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
664	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
665	}
666
667	TEST_F(TokenAnnotatorTest, UnderstandsVariables) {
668	auto Tokens =
669	annotate(Code: "inline bool var = is_integral_v<int> && is_signed_v<int>;");
670	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
671	EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_BinaryOperator);
672	}
673
674	TEST_F(TokenAnnotatorTest, UnderstandsVariableTemplates) {
675	auto Tokens =
676	annotate(Code: "template <typename T> "
677	"inline bool var = is_integral_v<int> && is_signed_v<int>;");
678	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
679	EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
680	}
681
682	TEST_F(TokenAnnotatorTest, UnderstandsTemplatesInMacros) {
683	auto Tokens =
684	annotate(Code: "#define FOO(typeName) \\\n"
685	" { #typeName, foo<FooType>(new foo<realClass>(#typeName)) }");
686	ASSERT_EQ(Tokens.size(), 27u) << Tokens;
687	EXPECT_TOKEN(Tokens[11], tok::less, TT_TemplateOpener);
688	EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
689	EXPECT_TOKEN(Tokens[17], tok::less, TT_TemplateOpener);
690	EXPECT_TOKEN(Tokens[19], tok::greater, TT_TemplateCloser);
691	}
692
693	TEST_F(TokenAnnotatorTest, UnderstandsGreaterAfterTemplateCloser) {
694	auto Tokens = annotate(Code: "if (std::tuple_size_v<T> > 0)");
695	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
696	EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);
697	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
698	EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
699	}
700
701	TEST_F(TokenAnnotatorTest, UnderstandsTernaryInTemplate) {
702	// IsExpression = false
703	auto Tokens = annotate(Code: "foo<true ? 1 : 2>();");
704	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
705	EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
706	EXPECT_TOKEN(Tokens[3], tok::question, TT_ConditionalExpr);
707	EXPECT_TOKEN(Tokens[5], tok::colon, TT_ConditionalExpr);
708	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
709
710	// IsExpression = true
711
712	Tokens = annotate(Code: "return foo<true ? 1 : 2>();");
713	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
714	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
715	EXPECT_TOKEN(Tokens[4], tok::question, TT_ConditionalExpr);
716	EXPECT_TOKEN(Tokens[6], tok::colon, TT_ConditionalExpr);
717	EXPECT_TOKEN(Tokens[8], tok::greater, TT_TemplateCloser);
718
719	Tokens = annotate(Code: "return foo<true ? 1 : 2>{};");
720	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
721	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
722	EXPECT_TOKEN(Tokens[4], tok::question, TT_ConditionalExpr);
723	EXPECT_TOKEN(Tokens[6], tok::colon, TT_ConditionalExpr);
724	EXPECT_TOKEN(Tokens[8], tok::greater, TT_TemplateCloser);
725	}
726
727	TEST_F(TokenAnnotatorTest, UnderstandsNonTemplateAngleBrackets) {
728	auto Tokens = annotate(Code: "return a < b && c > d;");
729	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
730	EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
731	EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);
732
733	Tokens = annotate(Code: "a < 0 ? b : a > 0 ? c : d;");
734	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
735	EXPECT_TOKEN(Tokens[1], tok::less, TT_BinaryOperator);
736	EXPECT_TOKEN(Tokens[7], tok::greater, TT_BinaryOperator);
737
738	Tokens = annotate(Code: "return A < B ? true : A > B;");
739	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
740	EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
741	EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
742
743	Tokens = annotate(Code: "return A < B ? true : A > B ? false : false;");
744	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
745	EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
746	EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
747
748	Tokens = annotate(Code: "return checklower ? a < b : a > b;");
749	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
750	EXPECT_TOKEN(Tokens[4], tok::less, TT_BinaryOperator);
751	EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
752
753	Tokens = annotate(Code: "return A < B != A > B;");
754	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
755	// FIXME:
756	// EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
757	// EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);
758
759	Tokens = annotate(Code: "ratio{-1, 2} < ratio{-1, 3} == -1 / 3 > -1 / 2;");
760	ASSERT_EQ(Tokens.size(), 27u) << Tokens;
761	EXPECT_TOKEN(Tokens[7], tok::less, TT_BinaryOperator);
762	EXPECT_TOKEN(Tokens[20], tok::greater, TT_BinaryOperator);
763
764	Tokens = annotate(Code: "bool foo = a < b && (c * d) > e;");
765	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
766	EXPECT_TOKEN(Tokens[4], tok::less, TT_BinaryOperator);
767	EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_BinaryOperator);
768	EXPECT_TOKEN(Tokens[9], tok::star, TT_BinaryOperator);
769	EXPECT_TOKEN(Tokens[12], tok::greater, TT_BinaryOperator);
770	}
771
772	TEST_F(TokenAnnotatorTest, UnderstandsTemplateTemplateParameters) {
773	auto Tokens = annotate(Code: "template <template <typename...> typename X,\n"
774	" template <typename...> class Y,\n"
775	" typename... T>\n"
776	"class A {};");
777	ASSERT_EQ(Tokens.size(), 28u) << Tokens;
778	EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
779	EXPECT_TOKEN(Tokens[3], tok::less, TT_TemplateOpener);
780	EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
781	EXPECT_FALSE(Tokens[6]->ClosesTemplateDeclaration);
782	EXPECT_TOKEN(Tokens[11], tok::less, TT_TemplateOpener);
783	EXPECT_TOKEN(Tokens[14], tok::greater, TT_TemplateCloser);
784	EXPECT_FALSE(Tokens[14]->ClosesTemplateDeclaration);
785	EXPECT_TOKEN(Tokens[16], tok::identifier, TT_Unknown);
786	EXPECT_TOKEN(Tokens[21], tok::greater, TT_TemplateCloser);
787	EXPECT_TRUE(Tokens[21]->ClosesTemplateDeclaration);
788	EXPECT_TOKEN(Tokens[23], tok::identifier, TT_ClassHeadName);
789	}
790
791	TEST_F(TokenAnnotatorTest, UnderstandsWhitespaceSensitiveMacros) {
792	FormatStyle Style = getLLVMStyle();
793	Style.WhitespaceSensitiveMacros.push_back(x: "FOO");
794
795	auto Tokens = annotate(Code: "FOO(1+2 )", Style);
796	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
797	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);
798
799	Tokens = annotate(Code: "FOO(a:b:c)", Style);
800	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
801	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);
802	}
803
804	TEST_F(TokenAnnotatorTest, UnderstandsDelete) {
805	auto Tokens = annotate(Code: "delete (void *)p;");
806	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
807	EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);
808
809	Tokens = annotate(Code: "delete[] (void *)p;");
810	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
811	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_CastRParen);
812
813	Tokens = annotate(Code: "delete[] /*comment*/ (void *)p;");
814	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
815	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
816
817	Tokens = annotate(Code: "delete[/*comment*/] (void *)p;");
818	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
819	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
820
821	Tokens = annotate(Code: "delete/*comment*/[] (void *)p;");
822	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
823	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
824	}
825
826	TEST_F(TokenAnnotatorTest, UnderstandsCasts) {
827	auto Tokens = annotate(Code: "(void)p;");
828	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
829	EXPECT_TOKEN(Tokens[2], tok::r_paren, TT_CastRParen);
830
831	Tokens = annotate(Code: "(uint32_t)&&label;");
832	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
833	EXPECT_TOKEN(Tokens[2], tok::r_paren, TT_CastRParen);
834	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_UnaryOperator);
835	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);
836
837	Tokens = annotate(Code: "auto x = (Foo)p;");
838	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
839	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_CastRParen);
840
841	Tokens = annotate(Code: "(std::vector<int>)p;");
842	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
843	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
844
845	Tokens = annotate(Code: "return (Foo)p;");
846	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
847	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);
848
849	Tokens = annotate(Code: "throw (Foo)p;");
850	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
851	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);
852
853	Tokens = annotate(Code: "#define FOO(x) (((uint64_t)(x) * BAR) / 100)");
854	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
855	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);
856	EXPECT_TOKEN(Tokens[13], tok::r_paren, TT_Unknown);
857	EXPECT_TOKEN(Tokens[14], tok::star, TT_BinaryOperator);
858
859	Tokens = annotate(Code: "#define foo(i) ((i) - bar)");
860	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
861	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_Unknown);
862	EXPECT_TOKEN(Tokens[10], tok::minus, TT_BinaryOperator);
863
864	Tokens = annotate(Code: "return (Foo) & 10;");
865	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
866	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_Unknown);
867	EXPECT_TOKEN(Tokens[4], tok::amp, TT_BinaryOperator);
868
869	Tokens = annotate(Code: "return (struct foo){};");
870	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
871	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
872	EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);
873
874	Tokens = annotate(Code: "#define FOO(bar) foo((uint64_t)&bar)");
875	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
876	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);
877	EXPECT_TOKEN(Tokens[11], tok::amp, TT_UnaryOperator);
878
879	Tokens = annotate(Code: "#define FOO(bar) foo((Foo) & bar)");
880	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
881	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_Unknown);
882	EXPECT_TOKEN(Tokens[11], tok::amp, TT_BinaryOperator);
883
884	Tokens = annotate(Code: "func((void (*)())&a);");
885	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
886	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionTypeLParen);
887	EXPECT_TOKEN(Tokens[5], tok::star, TT_PointerOrReference);
888	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_CastRParen);
889	EXPECT_TOKEN(Tokens[10], tok::amp, TT_UnaryOperator);
890
891	Tokens = annotate(Code: "int result = ((int)a) - b;");
892	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
893	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_CastRParen);
894	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
895	EXPECT_TOKEN(Tokens[9], tok::minus, TT_BinaryOperator);
896
897	Tokens = annotate(Code: "return (double)(foo(30)) - 15;");
898	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
899	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);
900	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_Unknown);
901	EXPECT_TOKEN(Tokens[10], tok::minus, TT_BinaryOperator);
902
903	Tokens = annotate(Code: "return (::Type)(1 + 2);");
904	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
905	EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);
906
907	Tokens = annotate(Code: "return (Namespace::Class)(1 + 2);");
908	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
909	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_CastRParen);
910
911	Tokens = annotate(Code: "return (Foo (*)(void *, Bar, ...))&foo;");
912	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
913	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionTypeLParen);
914	EXPECT_TOKEN(Tokens[14], tok::r_paren, TT_CastRParen);
915	EXPECT_TOKEN(Tokens[15], tok::amp, TT_UnaryOperator);
916
917	Tokens = annotate(Code: "return (Foo (Bar::*)())&Bar::foo;");
918	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
919	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionTypeLParen);
920	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);
921	EXPECT_TOKEN(Tokens[11], tok::amp, TT_UnaryOperator);
922
923	auto Style = getLLVMStyle();
924	Style.TypeNames.push_back(x: "Foo");
925	Tokens = annotate(Code: "#define FOO(bar) foo((Foo)&bar)", Style);
926	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
927	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_TypeName);
928	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);
929	EXPECT_TOKEN(Tokens[11], tok::amp, TT_UnaryOperator);
930	}
931
932	TEST_F(TokenAnnotatorTest, UnderstandsDynamicExceptionSpecifier) {
933	auto Tokens = annotate(Code: "void f() throw(int);");
934	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
935	EXPECT_TOKEN(Tokens[4], tok::kw_throw, TT_Unknown);
936	}
937
938	TEST_F(TokenAnnotatorTest, UnderstandsFunctionRefQualifiers) {
939	auto Tokens = annotate(Code: "void f() &;");
940	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
941	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
942
943	Tokens = annotate(Code: "void operator=(T) &&;");
944	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
945	EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference);
946
947	Tokens = annotate(Code: "template <typename T> void f() &;");
948	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
949	EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
950
951	Tokens = annotate(Code: "template <typename T> void operator=(T) &;");
952	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
953	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
954	}
955
956	TEST_F(TokenAnnotatorTest, UnderstandsOverloadedOperators) {
957	auto Tokens = annotate(Code: "x.operator+()");
958	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
959	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
960	EXPECT_TOKEN(Tokens[3], tok::plus, TT_OverloadedOperator);
961	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
962	Tokens = annotate(Code: "x.operator=()");
963	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
964	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
965	EXPECT_TOKEN(Tokens[3], tok::equal, TT_OverloadedOperator);
966	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
967	Tokens = annotate(Code: "x.operator+=()");
968	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
969	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
970	EXPECT_TOKEN(Tokens[3], tok::plusequal, TT_OverloadedOperator);
971	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
972	Tokens = annotate(Code: "x.operator,()");
973	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
974	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
975	EXPECT_TOKEN(Tokens[3], tok::comma, TT_OverloadedOperator);
976	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
977	Tokens = annotate(Code: "x.operator()()");
978	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
979	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
980	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperator);
981	EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_OverloadedOperator);
982	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
983	Tokens = annotate(Code: "x.operator[]()");
984	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
985	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
986	// EXPECT_TOKEN(Tokens[3], tok::l_square, TT_OverloadedOperator);
987	// EXPECT_TOKEN(Tokens[4], tok::r_square, TT_OverloadedOperator);
988	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
989	Tokens = annotate(Code: "x.operator\"\"_a()");
990	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
991	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
992	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
993	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
994	Tokens = annotate(Code: "x.operator\"\" _a()");
995	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
996	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
997	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
998	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
999	Tokens = annotate(Code: "x.operator\"\"if()");
1000	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1001	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
1002	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
1003	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
1004	Tokens = annotate(Code: "x.operator\"\"s()");
1005	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1006	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
1007	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
1008	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
1009	Tokens = annotate(Code: "x.operator\"\" s()");
1010	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1011	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
1012	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
1013	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
1014
1015	Tokens = annotate(Code: "int operator+(int);");
1016	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1017	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
1018	EXPECT_TOKEN(Tokens[2], tok::plus, TT_OverloadedOperator);
1019	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
1020	Tokens = annotate(Code: "auto operator=(T&) {}");
1021	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1022	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
1023	EXPECT_TOKEN(Tokens[2], tok::equal, TT_OverloadedOperator);
1024	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
1025	Tokens = annotate(Code: "auto operator()() {}");
1026	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1027	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
1028	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_OverloadedOperator);
1029	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_OverloadedOperator);
1030	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
1031
1032	Tokens = annotate(Code: "class Foo {\n"
1033	" int operator+(a* b);\n"
1034	"}");
1035	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1036	EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);
1037	EXPECT_TOKEN(Tokens[5], tok::plus, TT_OverloadedOperator);
1038	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen);
1039	EXPECT_TOKEN(Tokens[8], tok::star, TT_PointerOrReference);
1040
1041	Tokens = annotate(Code: "class Foo {\n"
1042	" int c = operator+(a * b);\n"
1043	"}");
1044	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1045	EXPECT_TOKEN(Tokens[6], tok::kw_operator, TT_Unknown);
1046	EXPECT_TOKEN(Tokens[7], tok::plus, TT_OverloadedOperator);
1047	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_OverloadedOperatorLParen);
1048	EXPECT_TOKEN(Tokens[10], tok::star, TT_BinaryOperator);
1049
1050	Tokens = annotate(Code: "void foo() {\n"
1051	" operator+(a * b);\n"
1052	"}");
1053	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1054	EXPECT_TOKEN(Tokens[5], tok::kw_operator, TT_Unknown);
1055	EXPECT_TOKEN(Tokens[6], tok::plus, TT_OverloadedOperator);
1056	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperatorLParen);
1057	EXPECT_TOKEN(Tokens[9], tok::star, TT_BinaryOperator);
1058
1059	Tokens = annotate(Code: "return operator+(a * b, c & d) + operator+(a && b && c);");
1060	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
1061	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_Unknown);
1062	EXPECT_TOKEN(Tokens[2], tok::plus, TT_OverloadedOperator);
1063	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
1064	EXPECT_TOKEN(Tokens[5], tok::star, TT_BinaryOperator);
1065	EXPECT_TOKEN(Tokens[9], tok::amp, TT_BinaryOperator);
1066	EXPECT_TOKEN(Tokens[13], tok::kw_operator, TT_Unknown);
1067	EXPECT_TOKEN(Tokens[14], tok::plus, TT_OverloadedOperator);
1068	EXPECT_TOKEN(Tokens[15], tok::l_paren, TT_OverloadedOperatorLParen);
1069	EXPECT_TOKEN(Tokens[17], tok::ampamp, TT_BinaryOperator);
1070	EXPECT_TOKEN(Tokens[19], tok::ampamp, TT_BinaryOperator);
1071
1072	Tokens = annotate(Code: "class Foo {\n"
1073	" void foo() {\n"
1074	" operator+(a * b);\n"
1075	" }\n"
1076	"}");
1077	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1078	EXPECT_TOKEN(Tokens[8], tok::kw_operator, TT_Unknown);
1079	EXPECT_TOKEN(Tokens[9], tok::plus, TT_OverloadedOperator);
1080	EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_OverloadedOperatorLParen);
1081	EXPECT_TOKEN(Tokens[12], tok::star, TT_BinaryOperator);
1082
1083	Tokens = annotate(Code: "std::vector<Foo> operator()(Foo &foo);");
1084	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1085	EXPECT_TOKEN(Tokens[3], tok::less, TT_TemplateOpener);
1086	EXPECT_TOKEN(Tokens[5], tok::greater, TT_TemplateCloser);
1087	EXPECT_TOKEN(Tokens[6], tok::kw_operator, TT_FunctionDeclarationName);
1088	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperator);
1089	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_OverloadedOperator);
1090	EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_OverloadedOperatorLParen);
1091	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
1092
1093	Tokens = annotate(Code: "decltype(auto) operator()(T &x);");
1094	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1095	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_TypeDeclarationParen);
1096	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_TypeDeclarationParen);
1097	EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);
1098	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperator);
1099	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_OverloadedOperator);
1100	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperatorLParen);
1101	EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
1102
1103	Tokens = annotate(Code: "friend ostream& ::operator<<(ostream& lhs, foo& rhs);");
1104	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1105	EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);
1106	EXPECT_TOKEN(Tokens[5], tok::lessless, TT_OverloadedOperator);
1107	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen);
1108	EXPECT_TOKEN(Tokens[8], tok::amp, TT_PointerOrReference);
1109	EXPECT_TOKEN(Tokens[12], tok::amp, TT_PointerOrReference);
1110
1111	Tokens = annotate(Code: "SomeLoooooooooooooooooType::Awaitable\n"
1112	"SomeLoooooooooooooooooType::operator co_await();");
1113	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1114	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_FunctionDeclarationName);
1115	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperatorLParen);
1116
1117	Tokens = annotate(Code: "using std::operator==;");
1118	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1119	// Not TT_FunctionDeclarationName.
1120	EXPECT_TOKEN(Tokens[3], tok::kw_operator, TT_Unknown);
1121	}
1122
1123	TEST_F(TokenAnnotatorTest, OverloadedOperatorInTemplate) {
1124	struct {
1125	const char *Text;
1126	tok::TokenKind Kind;
1127	} Operators[] = {{.Text: "+", .Kind: tok::plus},
1128	{.Text: "-", .Kind: tok::minus},
1129	// FIXME:
1130	// {"*", tok::star},
1131	{.Text: "/", .Kind: tok::slash},
1132	{.Text: "%", .Kind: tok::percent},
1133	{.Text: "^", .Kind: tok::caret},
1134	// FIXME:
1135	// {"&", tok::amp},
1136	{.Text: "|", .Kind: tok::pipe},
1137	{.Text: "~", .Kind: tok::tilde},
1138	{.Text: "!", .Kind: tok::exclaim},
1139	{.Text: "=", .Kind: tok::equal},
1140	// FIXME:
1141	// {"<", tok::less},
1142	{.Text: ">", .Kind: tok::greater},
1143	{.Text: "+=", .Kind: tok::plusequal},
1144	{.Text: "-=", .Kind: tok::minusequal},
1145	{.Text: "*=", .Kind: tok::starequal},
1146	{.Text: "/=", .Kind: tok::slashequal},
1147	{.Text: "%=", .Kind: tok::percentequal},
1148	{.Text: "^=", .Kind: tok::caretequal},
1149	{.Text: "&=", .Kind: tok::ampequal},
1150	{.Text: "|=", .Kind: tok::pipeequal},
1151	{.Text: "<<", .Kind: tok::lessless},
1152	{.Text: ">>", .Kind: tok::greatergreater},
1153	{.Text: ">>=", .Kind: tok::greatergreaterequal},
1154	{.Text: "<<=", .Kind: tok::lesslessequal},
1155	{.Text: "==", .Kind: tok::equalequal},
1156	{.Text: "!=", .Kind: tok::exclaimequal},
1157	{.Text: "<=", .Kind: tok::lessequal},
1158	{.Text: ">=", .Kind: tok::greaterequal},
1159	{.Text: "<=>", .Kind: tok::spaceship},
1160	{.Text: "&&", .Kind: tok::ampamp},
1161	{.Text: "||", .Kind: tok::pipepipe},
1162	{.Text: "++", .Kind: tok::plusplus},
1163	{.Text: "--", .Kind: tok::minusminus},
1164	{.Text: ",", .Kind: tok::comma},
1165	{.Text: "->*", .Kind: tok::arrowstar},
1166	{.Text: "->", .Kind: tok::arrow}};
1167
1168	for (const auto &Operator : Operators) {
1169	std::string Input("C<&operator");
1170	Input += Operator.Text;
1171	Input += " > a;";
1172	auto Tokens = annotate(Code: std::string(Input));
1173	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1174	EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
1175	EXPECT_TOKEN(Tokens[4], Operator.Kind, TT_OverloadedOperator);
1176	EXPECT_TOKEN(Tokens[5], tok::greater, TT_TemplateCloser);
1177	}
1178
1179	auto Tokens = annotate(Code: "C<&operator< <X>> lt;");
1180	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1181	EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
1182	EXPECT_TOKEN(Tokens[4], tok::less, TT_OverloadedOperator);
1183	EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);
1184	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
1185	EXPECT_TOKEN(Tokens[8], tok::greater, TT_TemplateCloser);
1186	}
1187
1188	TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
1189	auto Tokens = annotate(Code: "template <typename T>\n"
1190	"concept C = (Foo && Bar) && (Bar && Baz);");
1191
1192	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1193	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
1194	EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
1195	EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator);
1196
1197	Tokens = annotate(Code: "template <typename T>\n"
1198	"concept C = Foo && !Bar;");
1199
1200	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1201	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
1202	EXPECT_TOKEN(Tokens[10], tok::exclaim, TT_UnaryOperator);
1203
1204	Tokens = annotate(Code: "template <typename T>\n"
1205	"concept C = requires(T t) {\n"
1206	" { t.foo() };\n"
1207	"} && Bar<T> && Baz<T>;");
1208	ASSERT_EQ(Tokens.size(), 35u) << Tokens;
1209	EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
1210	EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
1211	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1212	EXPECT_TOKEN(Tokens[23], tok::ampamp, TT_BinaryOperator);
1213	EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);
1214
1215	Tokens = annotate(Code: "template<typename T>\n"
1216	"requires C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>\n"
1217	"struct Foo;");
1218	ASSERT_EQ(Tokens.size(), 36u) << Tokens;
1219	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1220	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
1221	EXPECT_EQ(Tokens[6]->FakeLParens.size(), 1u);
1222	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
1223	EXPECT_TOKEN(Tokens[16], tok::pipepipe, TT_BinaryOperator);
1224	EXPECT_TOKEN(Tokens[21], tok::ampamp, TT_BinaryOperator);
1225	EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator);
1226	// Not TT_TrailingAnnotation.
1227	EXPECT_TOKEN(Tokens[28], tok::identifier, TT_Unknown);
1228	EXPECT_TOKEN(Tokens[31], tok::greater, TT_TemplateCloser);
1229	EXPECT_EQ(Tokens[31]->FakeRParens, 1u);
1230	EXPECT_TRUE(Tokens[31]->ClosesRequiresClause);
1231	// Not TT_TrailingAnnotation.
1232	EXPECT_TOKEN(Tokens[33], tok::identifier, TT_Unknown);
1233
1234	Tokens =
1235	annotate(Code: "template <typename T>\n"
1236	" requires(C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>)\n"
1237	"struct Foo;");
1238	ASSERT_EQ(Tokens.size(), 38u) << Tokens;
1239	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1240	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_Unknown);
1241	EXPECT_EQ(Tokens[7]->FakeLParens.size(), 1u);
1242	EXPECT_TOKEN(Tokens[11], tok::ampamp, TT_BinaryOperator);
1243	EXPECT_TOKEN(Tokens[17], tok::pipepipe, TT_BinaryOperator);
1244	EXPECT_TOKEN(Tokens[22], tok::ampamp, TT_BinaryOperator);
1245	EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);
1246	EXPECT_TOKEN(Tokens[32], tok::greater, TT_TemplateCloser);
1247	EXPECT_EQ(Tokens[32]->FakeRParens, 1u);
1248	EXPECT_TOKEN(Tokens[33], tok::r_paren, TT_Unknown);
1249	EXPECT_TRUE(Tokens[33]->ClosesRequiresClause);
1250	EXPECT_TOKEN(Tokens[35], tok::identifier, TT_Unknown);
1251
1252	Tokens = annotate(Code: "template <typename T>\n"
1253	"void foo(T) noexcept requires Bar<T>;");
1254	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1255	EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause);
1256
1257	Tokens = annotate(Code: "template <typename T>\n"
1258	"requires Bar<T> || Baz<T>\n"
1259	"auto foo(T) -> int;");
1260	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
1261	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1262	EXPECT_EQ(Tokens[11]->FakeLParens.size(), 0u);
1263	EXPECT_TRUE(Tokens[14]->ClosesRequiresClause);
1264	EXPECT_TOKEN(Tokens[20], tok::arrow, TT_TrailingReturnArrow);
1265
1266	Tokens = annotate(Code: "template <typename T>\n"
1267	"requires Bar<T>\n"
1268	"bool foo(T) { return false; }");
1269	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1270	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1271	EXPECT_TRUE(Tokens[9]->ClosesRequiresClause);
1272	EXPECT_TOKEN(Tokens[11], tok::identifier, TT_FunctionDeclarationName);
1273	EXPECT_TOKEN(Tokens[12], tok::l_paren, TT_FunctionDeclarationLParen);
1274
1275	Tokens = annotate(Code: "template <typename T>\n"
1276	"requires Bar<T>\n"
1277	"decltype(auto) foo(T) { return false; }");
1278	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
1279	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1280	EXPECT_TRUE(Tokens[9]->ClosesRequiresClause);
1281	EXPECT_TOKEN(Tokens[14], tok::identifier, TT_FunctionDeclarationName);
1282	EXPECT_TOKEN(Tokens[15], tok::l_paren, TT_FunctionDeclarationLParen);
1283
1284	Tokens = annotate(Code: "template <typename T>\n"
1285	"struct S {\n"
1286	" void foo() const requires Bar<T>;\n"
1287	" void bar() const & requires Baz<T>;\n"
1288	" void bar() && requires Baz2<T>;\n"
1289	" void baz() const & noexcept requires Baz<T>;\n"
1290	" void baz() && noexcept requires Baz2<T>;\n"
1291	"};\n"
1292	"\n"
1293	"void S::bar() const & requires Baz<T> { }");
1294	ASSERT_EQ(Tokens.size(), 85u) << Tokens;
1295	EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
1296	EXPECT_TOKEN(Tokens[24], tok::amp, TT_PointerOrReference);
1297	EXPECT_TOKEN(Tokens[25], tok::kw_requires, TT_RequiresClause);
1298	EXPECT_TOKEN(Tokens[35], tok::ampamp, TT_PointerOrReference);
1299	EXPECT_TOKEN(Tokens[36], tok::kw_requires, TT_RequiresClause);
1300	EXPECT_TOKEN(Tokens[47], tok::amp, TT_PointerOrReference);
1301	EXPECT_TOKEN(Tokens[49], tok::kw_requires, TT_RequiresClause);
1302	EXPECT_TOKEN(Tokens[59], tok::ampamp, TT_PointerOrReference);
1303	EXPECT_TOKEN(Tokens[61], tok::kw_requires, TT_RequiresClause);
1304	EXPECT_TOKEN(Tokens[76], tok::amp, TT_PointerOrReference);
1305	EXPECT_TOKEN(Tokens[77], tok::kw_requires, TT_RequiresClause);
1306
1307	Tokens = annotate(Code: "void Class::member() && requires(Constant) {}");
1308	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1309	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1310
1311	Tokens = annotate(Code: "void Class::member() && requires(Constant<T>) {}");
1312	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1313	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1314
1315	Tokens =
1316	annotate(Code: "void Class::member() && requires(Namespace::Constant<T>) {}");
1317	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1318	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1319
1320	Tokens = annotate(Code: "void Class::member() && requires(typename "
1321	"Namespace::Outer<T>::Inner::Constant) {}");
1322	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
1323	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1324
1325	Tokens = annotate(Code: "struct [[nodiscard]] zero_t {\n"
1326	" template<class T>\n"
1327	" requires requires { number_zero_v<T>; }\n"
1328	" [[nodiscard]] constexpr operator T() const { "
1329	"return number_zero_v<T>; }\n"
1330	"};");
1331	ASSERT_EQ(Tokens.size(), 44u) << Tokens;
1332	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);
1333	EXPECT_TOKEN(Tokens[11], tok::identifier, TT_Unknown);
1334	EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
1335	EXPECT_TOKEN(Tokens[14], tok::kw_requires, TT_RequiresExpression);
1336	EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_RequiresExpressionLBrace);
1337	EXPECT_TOKEN(Tokens[21], tok::r_brace, TT_Unknown);
1338	EXPECT_EQ(Tokens[21]->MatchingParen, Tokens[15]);
1339	EXPECT_TRUE(Tokens[21]->ClosesRequiresClause);
1340
1341	Tokens =
1342	annotate(Code: "template <class A, class B> concept C ="
1343	"std::same_as<std::iter_value_t<A>, std::iter_value_t<B>>;");
1344	ASSERT_EQ(Tokens.size(), 31u) << Tokens;
1345	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
1346	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
1347	EXPECT_TOKEN(Tokens[8], tok::kw_concept, TT_Unknown);
1348	EXPECT_TOKEN(Tokens[14], tok::less, TT_TemplateOpener);
1349	EXPECT_TOKEN(Tokens[18], tok::less, TT_TemplateOpener);
1350	EXPECT_TOKEN(Tokens[20], tok::greater, TT_TemplateCloser);
1351	EXPECT_TOKEN(Tokens[25], tok::less, TT_TemplateOpener);
1352	EXPECT_TOKEN(Tokens[27], tok::greater, TT_TemplateCloser);
1353	EXPECT_TOKEN(Tokens[28], tok::greater, TT_TemplateCloser);
1354
1355	Tokens = annotate(Code: "auto bar() -> int requires(is_integral_v<T>) {}");
1356	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1357	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1358
1359	Tokens = annotate(Code: "auto bar() -> void requires(is_integral_v<T>) {}");
1360	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1361	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1362
1363	Tokens = annotate(Code: "auto bar() -> MyType requires(is_integral_v<T>) {}");
1364	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1365	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1366
1367	Tokens =
1368	annotate(Code: "auto bar() -> SOME_MACRO_TYPE requires(is_integral_v<T>) {}");
1369	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1370	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1371
1372	Tokens =
1373	annotate(Code: "auto bar() -> qualified::type requires(is_integral_v<T>) {}");
1374	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1375	EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresClause);
1376
1377	Tokens =
1378	annotate(Code: "auto bar() -> Template<type> requires(is_integral_v<T>) {}");
1379	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1380	EXPECT_TOKEN(Tokens[9], tok::kw_requires, TT_RequiresClause);
1381
1382	Tokens = annotate(Code: "void foo() requires((A<T>) && C) {}");
1383	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1384	EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1385	EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_BinaryOperator);
1386
1387	Tokens = annotate(Code: "void foo() requires(((A<T>) && C)) {}");
1388	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1389	EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1390	EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
1391
1392	Tokens = annotate(Code: "void foo() requires([](T&&){}(t)) {}");
1393	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1394	EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1395	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
1396
1397	Tokens = annotate(Code: "void foo() requires([](T&& u){}(t)) {}");
1398	ASSERT_EQ(Tokens.size(), 22u) << Tokens;
1399	EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1400	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
1401
1402	Tokens = annotate(Code: "void f() & requires(true) {}");
1403	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1404	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
1405	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1406
1407	Tokens = annotate(Code: "void f() & requires(C<true, true>) {}");
1408	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1409	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
1410	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1411
1412	Tokens = annotate(Code: "template <typename T>\n"
1413	"concept C = (!Foo<T>) && Bar;");
1414	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1415	EXPECT_TOKEN(Tokens[15], tok::ampamp, TT_BinaryOperator);
1416
1417	Tokens = annotate(Code: "void f() & requires(C<decltype(x)>) {}");
1418	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1419	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
1420	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1421
1422	Tokens = annotate(Code: "auto f() -> int& requires(C<decltype(x)>) {}");
1423	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1424	EXPECT_TOKEN(Tokens[6], tok::amp, TT_PointerOrReference);
1425	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1426
1427	Tokens = annotate(Code: "bool x = t && requires(decltype(t) x) { x.foo(); };");
1428	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
1429	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresExpression);
1430
1431	Tokens = annotate(Code: "bool x = t && requires(Foo<decltype(t)> x) { x.foo(); };");
1432	ASSERT_EQ(Tokens.size(), 26u) << Tokens;
1433	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresExpression);
1434
1435	Tokens = annotate(Code: "bool x = t && requires(Foo<C1 || C2> x) { x.foo(); };");
1436	ASSERT_EQ(Tokens.size(), 25u) << Tokens;
1437	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresExpression);
1438
1439	// Second function definition is required due to lookahead
1440	Tokens = annotate(Code: "void f() &\n"
1441	" requires(n == 1)\n"
1442	"{}\n"
1443	"void g();");
1444	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1445	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
1446	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1447
1448	Tokens = annotate(Code: "void foo() &&\n"
1449	" requires(!bar)\n"
1450	"{\n"
1451	" baz();\n"
1452	"}");
1453	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1454	EXPECT_TOKEN(Tokens[4], tok::ampamp, TT_PointerOrReference);
1455	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1456
1457	Tokens = annotate(Code: "auto foo() -> auto *\n"
1458	" requires(not bar)\n"
1459	"{\n"
1460	" return baz;\n"
1461	"}");
1462	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1463	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1464	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);
1465	}
1466
1467	TEST_F(TokenAnnotatorTest, UnderstandsRequiresExpressions) {
1468	auto Tokens = annotate(Code: "bool b = requires(int i) { i + 5; };");
1469	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1470	EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1471	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1472	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1473
1474	Tokens = annotate(Code: "if (requires(int i) { i + 5; }) return;");
1475	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1476	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1477	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1478	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_RequiresExpressionLBrace);
1479
1480	Tokens = annotate(Code: "if (func() && requires(int i) { i + 5; }) return;");
1481	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1482	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresExpression);
1483	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_RequiresExpressionLParen);
1484	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_RequiresExpressionLBrace);
1485
1486	Tokens = annotate(Code: "foo(requires(const T t) {});");
1487	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1488	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1489	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1490	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1491
1492	Tokens = annotate(Code: "foo(requires(const int t) {});");
1493	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1494	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1495	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1496	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1497
1498	Tokens = annotate(Code: "foo(requires(const T t) {});");
1499	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1500	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1501	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1502	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1503
1504	Tokens = annotate(Code: "foo(requires(int const* volatile t) {});");
1505	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1506	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1507	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1508	EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);
1509	EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
1510
1511	Tokens = annotate(Code: "foo(requires(T const* volatile t) {});");
1512	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1513	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1514	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1515	EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);
1516	EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
1517
1518	Tokens = annotate(Code: "foo(requires(T& t) {});");
1519	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1520	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1521	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1522	EXPECT_TOKEN(Tokens[5], tok::amp, TT_PointerOrReference);
1523	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1524
1525	Tokens = annotate(Code: "foo(requires(T&& t) {});");
1526	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1527	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1528	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1529	EXPECT_TOKEN(Tokens[5], tok::ampamp, TT_PointerOrReference);
1530	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1531
1532	Tokens = annotate(Code: "bool foo = requires(T& t) {};");
1533	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1534	EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1535	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1536	EXPECT_TOKEN(Tokens[6], tok::amp, TT_PointerOrReference);
1537	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_RequiresExpressionLBrace);
1538
1539	Tokens = annotate(Code: "bool foo = requires(T&& t) {};");
1540	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1541	EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1542	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1543	EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference);
1544	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_RequiresExpressionLBrace);
1545
1546	Tokens =
1547	annotate(Code: "foo(requires(const typename Outer<T>::Inner * const t) {});");
1548	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1549	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1550	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1551	EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
1552	EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_RequiresExpressionLBrace);
1553
1554	Tokens = annotate(Code: "template <typename T>\n"
1555	"concept C = requires(T T) {\n"
1556	" requires Bar<T> && Foo<T>;\n"
1557	"};");
1558	ASSERT_EQ(Tokens.size(), 28u) << Tokens;
1559	EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
1560	EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
1561	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1562	EXPECT_TOKEN(Tokens[14], tok::kw_requires,
1563	TT_RequiresClauseInARequiresExpression);
1564
1565	Tokens = annotate(Code: "template <typename T>\n"
1566	"concept C = requires(T T) {\n"
1567	" { t.func() } -> std::same_as<int>;"
1568	" requires Bar<T> && Foo<T>;\n"
1569	"};");
1570	ASSERT_EQ(Tokens.size(), 43u) << Tokens;
1571	EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
1572	EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
1573	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1574	EXPECT_TOKEN(Tokens[29], tok::kw_requires,
1575	TT_RequiresClauseInARequiresExpression);
1576
1577	// Invalid Code, but we don't want to crash. See http://llvm.org/PR54350.
1578	Tokens = annotate(Code: "bool r10 = requires (struct new_struct { int x; } s) { "
1579	"requires true; };");
1580	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1581	EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1582	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1583	EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_RequiresExpressionLBrace);
1584
1585	Tokens = annotate(Code: "bool foo = requires(C<true, true> c) {\n"
1586	" { c.foo(); }\n"
1587	"};");
1588	ASSERT_EQ(Tokens.size(), 25u) << Tokens;
1589	EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1590	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1591	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1592	}
1593
1594	TEST_F(TokenAnnotatorTest, UnderstandsPragmaRegion) {
1595	// Everything after #pragma region should be ImplicitStringLiteral
1596	auto Tokens = annotate(Code: "#pragma region Foo(Bar: Hello)");
1597	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1598	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_ImplicitStringLiteral);
1599	EXPECT_TOKEN(Tokens[6], tok::colon, TT_ImplicitStringLiteral);
1600	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_ImplicitStringLiteral);
1601
1602	// Make sure it's annotated correctly inside a function as well
1603	Tokens = annotate(Code: "void test(){\n#pragma region Foo(Bar: Hello)\n}");
1604	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1605	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_ImplicitStringLiteral);
1606	EXPECT_TOKEN(Tokens[11], tok::colon, TT_ImplicitStringLiteral);
1607	EXPECT_TOKEN(Tokens[12], tok::identifier, TT_ImplicitStringLiteral);
1608	}
1609
1610	TEST_F(TokenAnnotatorTest, RequiresDoesNotChangeParsingOfTheRest) {
1611	const char *BaseCode = nullptr;
1612	const char *ConstrainedCode = nullptr;
1613	auto BaseTokenCount = 0u;
1614	auto RequiresTokenCount = 0u;
1615	auto PrefixTokenCount = 0u;
1616
1617	auto TestRequires = [&](int Line) {
1618	const auto BaseTokens = annotate(Code: BaseCode);
1619	const auto ConstrainedTokens = annotate(Code: ConstrainedCode);
1620
1621	#define LINE " (Line " << Line << ')'
1622
1623	ASSERT_EQ(BaseTokens.size(), BaseTokenCount) << BaseTokens << LINE;
1624	ASSERT_EQ(ConstrainedTokens.size(), BaseTokenCount + RequiresTokenCount)
1625	<< LINE;
1626
1627	for (auto I = 0u; I < BaseTokenCount; ++I) {
1628	EXPECT_EQ(
1629	*BaseTokens[I],
1630	*ConstrainedTokens[I < PrefixTokenCount ? I : I + RequiresTokenCount])
1631	<< I << LINE;
1632	}
1633
1634	#undef LINE
1635	};
1636
1637	BaseCode = "template<typename T>\n"
1638	"T Pi = 3.14;";
1639	ConstrainedCode = "template<typename T>\n"
1640	" requires Foo<T>\n"
1641	"T Pi = 3.14;";
1642	BaseTokenCount = 11;
1643	RequiresTokenCount = 5;
1644	PrefixTokenCount = 5;
1645	TestRequires(__LINE__);
1646
1647	BaseCode = "template<typename T>\n"
1648	"struct Bar;";
1649	ConstrainedCode = "template<typename T>\n"
1650	" requires Foo<T>\n"
1651	"struct Bar;";
1652	BaseTokenCount = 9;
1653	TestRequires(__LINE__);
1654
1655	BaseCode = "template<typename T>\n"
1656	"struct Bar {\n"
1657	" T foo();\n"
1658	" T bar();\n"
1659	"};";
1660	ConstrainedCode = "template<typename T>\n"
1661	" requires Foo<T>\n"
1662	"struct Bar {\n"
1663	" T foo();\n"
1664	" T bar();\n"
1665	"};";
1666	BaseTokenCount = 21;
1667	TestRequires(__LINE__);
1668
1669	BaseCode = "template<typename T>\n"
1670	"Bar(T) -> Bar<T>;";
1671	ConstrainedCode = "template<typename T>\n"
1672	" requires Foo<T>\n"
1673	"Bar(T) -> Bar<T>;";
1674	BaseTokenCount = 16;
1675	TestRequires(__LINE__);
1676
1677	BaseCode = "template<typename T>\n"
1678	"T foo();";
1679	ConstrainedCode = "template<typename T>\n"
1680	" requires Foo<T>\n"
1681	"T foo();";
1682	BaseTokenCount = 11;
1683	TestRequires(__LINE__);
1684
1685	BaseCode = "template<typename T>\n"
1686	"T foo() {\n"
1687	" auto bar = baz();\n"
1688	" return bar + T{};\n"
1689	"}";
1690	ConstrainedCode = "template<typename T>\n"
1691	" requires Foo<T>\n"
1692	"T foo() {\n"
1693	" auto bar = baz();\n"
1694	" return bar + T{};\n"
1695	"}";
1696	BaseTokenCount = 26;
1697	TestRequires(__LINE__);
1698
1699	BaseCode = "template<typename T>\n"
1700	"T foo();";
1701	ConstrainedCode = "template<typename T>\n"
1702	"T foo() requires Foo<T>;";
1703	BaseTokenCount = 11;
1704	PrefixTokenCount = 9;
1705	TestRequires(__LINE__);
1706
1707	BaseCode = "template<typename T>\n"
1708	"T foo() {\n"
1709	" auto bar = baz();\n"
1710	" return bar + T{};\n"
1711	"}";
1712	ConstrainedCode = "template<typename T>\n"
1713	"T foo() requires Foo<T> {\n"
1714	" auto bar = baz();\n"
1715	" return bar + T{};\n"
1716	"}";
1717	BaseTokenCount = 26;
1718	TestRequires(__LINE__);
1719
1720	BaseCode = "template<typename T>\n"
1721	"T foo();";
1722	ConstrainedCode = "template<typename T>\n"
1723	" requires(Foo<T>)\n"
1724	"T foo();";
1725	BaseTokenCount = 11;
1726	RequiresTokenCount = 7;
1727	PrefixTokenCount = 5;
1728	TestRequires(__LINE__);
1729
1730	BaseCode = "template<typename T>\n"
1731	"Bar(T) -> Bar<typename T::I>;";
1732	ConstrainedCode = "template<typename T>\n"
1733	" requires requires(T &&t) {\n"
1734	" typename T::I;\n"
1735	" }\n"
1736	"Bar(T) -> Bar<typename T::I>;";
1737	BaseTokenCount = 19;
1738	RequiresTokenCount = 14;
1739	PrefixTokenCount = 5;
1740	TestRequires(__LINE__);
1741
1742	BaseCode = "struct [[nodiscard]] zero_t {\n"
1743	" template<class T>\n"
1744	" [[nodiscard]] constexpr operator T() const { return v<T>; }\n"
1745	"};";
1746	ConstrainedCode =
1747	"struct [[nodiscard]] zero_t {\n"
1748	" template<class T>\n"
1749	" requires requires { v<T>; }\n"
1750	" [[nodiscard]] constexpr operator T() const { return v<T>; }\n"
1751	"};";
1752	BaseTokenCount = 35;
1753	RequiresTokenCount = 9;
1754	PrefixTokenCount = 13;
1755	TestRequires(__LINE__);
1756
1757	BaseCode = "constexpr Foo(Foo const &other)\n"
1758	" : value{other.value} {\n"
1759	" do_magic();\n"
1760	" do_more_magic();\n"
1761	"}";
1762	ConstrainedCode = "constexpr Foo(Foo const &other)\n"
1763	" requires std::is_copy_constructible<T>\n"
1764	" : value{other.value} {\n"
1765	" do_magic();\n"
1766	" do_more_magic();\n"
1767	"}";
1768	BaseTokenCount = 26;
1769	RequiresTokenCount = 7;
1770	PrefixTokenCount = 8;
1771	TestRequires(__LINE__);
1772
1773	BaseCode = "constexpr Foo(Foo const &other)\n"
1774	" : value{other.value} {\n"
1775	" do_magic();\n"
1776	" do_more_magic();\n"
1777	"}";
1778	ConstrainedCode = "constexpr Foo(Foo const &other)\n"
1779	" requires (std::is_copy_constructible<T>)\n"
1780	" : value{other.value} {\n"
1781	" do_magic();\n"
1782	" do_more_magic();\n"
1783	"}";
1784	RequiresTokenCount = 9;
1785	TestRequires(__LINE__);
1786
1787	BaseCode = "template<typename T>\n"
1788	"ANNOTATE(\"S\"\n"
1789	" \"S\")\n"
1790	"void foo();";
1791	ConstrainedCode = "template<typename T>\n"
1792	" requires(true)\n"
1793	"ANNOTATE(\"S\"\n"
1794	" \"S\")\n"
1795	"void foo();";
1796	BaseTokenCount = 16;
1797	RequiresTokenCount = 4;
1798	PrefixTokenCount = 5;
1799	TestRequires(__LINE__);
1800	}
1801
1802	TEST_F(TokenAnnotatorTest, UnderstandsAsm) {
1803	auto Tokens = annotate(Code: "__asm{\n"
1804	"\"a\":\n"
1805	": x\n"
1806	":};");
1807	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1808	EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1809	EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_InlineASMBrace);
1810	EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
1811	EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
1812	EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
1813	EXPECT_TOKEN(Tokens[7], tok::r_brace, TT_InlineASMBrace);
1814
1815	Tokens = annotate(Code: "__asm(\n"
1816	"\"a\":\n"
1817	": x\n"
1818	":);");
1819	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1820	EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1821	EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
1822	EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
1823	EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
1824
1825	Tokens = annotate(Code: "asm volatile (\n"
1826	"\"a_label:\"\n"
1827	":\n"
1828	": x\n"
1829	":);");
1830	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1831	EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1832	EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
1833	EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
1834	EXPECT_TOKEN(Tokens[7], tok::colon, TT_InlineASMColon);
1835
1836	Tokens = annotate(Code: "__asm__(\n"
1837	"\"a_label:\"\n"
1838	": x\n"
1839	":\n"
1840	": y);");
1841	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1842	EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1843	EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
1844	EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
1845	EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
1846
1847	Tokens = annotate(Code: "__asm volatile (\n"
1848	"\"a_label:\"\n"
1849	"\"a b c(%%x)\"\n"
1850	":\n"
1851	": x\n"
1852	":);");
1853	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1854	EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1855	EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
1856	EXPECT_TOKEN(Tokens[6], tok::colon, TT_InlineASMColon);
1857	EXPECT_TOKEN(Tokens[8], tok::colon, TT_InlineASMColon);
1858
1859	Tokens = annotate(Code: "asm(\n"
1860	"\"insn\"\n"
1861	": \"=r\" (var1), \"=&r\" (value)\n"
1862	":\n"
1863	": \"memory\");");
1864	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1865	EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1866	EXPECT_TOKEN(Tokens[3], tok::colon, TT_InlineASMColon);
1867	EXPECT_TOKEN(Tokens[13], tok::colon, TT_InlineASMColon);
1868	EXPECT_TOKEN(Tokens[14], tok::colon, TT_InlineASMColon);
1869
1870	Tokens = annotate(Code: "__asm__ volatile (\n"
1871	"\"ldr r1, [r0, %%[sym]]\"\n"
1872	":\n"
1873	": [sym] \"J\" (aaaaa(aaaa, aaaa))\n"
1874	");");
1875	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1876	EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1877	EXPECT_TOKEN(Tokens[4], tok::colon, TT_InlineASMColon);
1878	EXPECT_TOKEN(Tokens[5], tok::colon, TT_InlineASMColon);
1879	EXPECT_TOKEN(Tokens[6], tok::l_square, TT_InlineASMSymbolicNameLSquare);
1880	}
1881
1882	TEST_F(TokenAnnotatorTest, UnderstandsObjCBlock) {
1883	auto Tokens = annotate(Code: "int (^)() = ^ ()\n"
1884	" external_source_symbol() { //\n"
1885	" return 1;\n"
1886	"};");
1887	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1888	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ObjCBlockLParen);
1889	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_ObjCBlockLBrace);
1890
1891	Tokens = annotate(Code: "int *p = ^int*(){ //\n"
1892	" return nullptr;\n"
1893	"}();");
1894	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1895	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_ObjCBlockLBrace);
1896
1897	Tokens = annotate(Code: "id (^block)(Foo *a) = ^id _Nullable(Foo *_Nullable a) {\n"
1898	" return a;\n"
1899	"};");
1900	ASSERT_EQ(Tokens.size(), 27u) << Tokens;
1901	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown); // Not CtorDtorDeclName.
1902	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ObjCBlockLParen);
1903	}
1904
1905	TEST_F(TokenAnnotatorTest, UnderstandsObjCMethodExpr) {
1906	auto Tokens = annotate(Code: "void f() {\n"
1907	" //\n"
1908	" BOOL a = [b.c n] > 1;\n"
1909	"}");
1910	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1911	EXPECT_TOKEN(Tokens[9], tok::l_square, TT_ObjCMethodExpr);
1912	EXPECT_TOKEN(Tokens[15], tok::greater, TT_BinaryOperator);
1913	}
1914
1915	TEST_F(TokenAnnotatorTest, UnderstandsObjCMethodDecl) {
1916	auto Tokens = annotate(Code: "/**/ - (void)foo;");
1917	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1918	EXPECT_TOKEN(Tokens[1], tok::minus, TT_ObjCMethodSpecifier);
1919	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_SelectorName);
1920	}
1921
1922	TEST_F(TokenAnnotatorTest, UnderstandsLambdas) {
1923	auto Tokens = annotate(Code: "[]() constexpr {}");
1924	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1925	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1926	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1927	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1928
1929	Tokens = annotate(Code: "[]() consteval {}");
1930	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1931	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1932	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1933	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1934
1935	Tokens = annotate(Code: "[]() mutable {}");
1936	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1937	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1938	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1939	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1940
1941	Tokens = annotate(Code: "[]() static {}");
1942	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1943	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1944	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1945	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1946
1947	Tokens = annotate(Code: "[]() -> auto {}");
1948	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1949	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1950	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1951	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
1952	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);
1953
1954	Tokens = annotate(Code: "[]() -> auto & {}");
1955	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1956	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1957	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1958	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
1959	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1960
1961	Tokens = annotate(Code: "[]() -> auto * {}");
1962	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1963	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1964	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_LambdaDefinitionLParen);
1965	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
1966	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1967
1968	Tokens = annotate(Code: "[] {}");
1969	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
1970	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1971	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_LambdaLBrace);
1972
1973	Tokens = annotate(Code: "[] noexcept {}");
1974	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
1975	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1976	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_LambdaLBrace);
1977
1978	Tokens = annotate(Code: "[] -> auto {}");
1979	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1980	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1981	EXPECT_TOKEN(Tokens[2], tok::arrow, TT_LambdaArrow);
1982	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_LambdaLBrace);
1983
1984	Tokens = annotate(Code: "[] -> struct S { return {}; }");
1985	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1986	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1987	EXPECT_TOKEN(Tokens[2], tok::arrow, TT_LambdaArrow);
1988	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);
1989	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1990
1991	Tokens = annotate(Code: "foo([&](u32 bar) __attribute__((attr)) -> void {});");
1992	ASSERT_EQ(Tokens.size(), 22u) << Tokens;
1993	EXPECT_TOKEN(Tokens[2], tok::l_square, TT_LambdaLSquare);
1994	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_LambdaDefinitionLParen);
1995	EXPECT_TOKEN(Tokens[15], tok::arrow, TT_LambdaArrow);
1996	EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);
1997
1998	Tokens = annotate(Code: "[] <typename T> () {}");
1999	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2000	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2001	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2002	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);
2003	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_LambdaLBrace);
2004
2005	Tokens = annotate(Code: "[] <typename T> {}");
2006	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2007	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2008	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2009	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);
2010
2011	Tokens = annotate(Code: "[] <typename... T> () {}");
2012	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2013	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2014	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2015	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_LambdaDefinitionLParen);
2016	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
2017
2018	Tokens = annotate(Code: "[] <typename... T> {}");
2019	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2020	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2021	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2022	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
2023
2024	Tokens = annotate(Code: "[] <int... T> () {}");
2025	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2026	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2027	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2028	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_LambdaDefinitionLParen);
2029	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
2030
2031	Tokens = annotate(Code: "[] <int... T> {}");
2032	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2033	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2034	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2035	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
2036
2037	Tokens = annotate(Code: "[] <Foo... T> () {}");
2038	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2039	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2040	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2041	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_LambdaDefinitionLParen);
2042	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
2043
2044	Tokens = annotate(Code: "[] <Foo... T> {}");
2045	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2046	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2047	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2048	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
2049
2050	// Lambdas with a requires-clause
2051	Tokens = annotate(Code: "[] <typename T> (T t) requires Bar<T> {}");
2052	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
2053	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2054	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2055	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);
2056	EXPECT_TOKEN(Tokens[10], tok::kw_requires, TT_RequiresClause);
2057	EXPECT_TRUE(Tokens[14]->ClosesRequiresClause);
2058	EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
2059
2060	Tokens = annotate(Code: "[] <typename T> (T &&t) requires Bar<T> {}");
2061	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
2062	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2063	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2064	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);
2065	EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_PointerOrReference);
2066	EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause);
2067	EXPECT_TRUE(Tokens[15]->ClosesRequiresClause);
2068	EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_LambdaLBrace);
2069
2070	Tokens = annotate(Code: "[] <typename T> (T t) requires Foo<T> || Bar<T> {}");
2071	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
2072	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2073	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2074	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);
2075	EXPECT_TOKEN(Tokens[10], tok::kw_requires, TT_RequiresClause);
2076	EXPECT_TRUE(Tokens[19]->ClosesRequiresClause);
2077	EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
2078
2079	Tokens = annotate(Code: "[] <typename T> (T t) -> T requires Bar<T> {}");
2080	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
2081	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2082	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2083	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_LambdaDefinitionLParen);
2084	EXPECT_TOKEN(Tokens[10], tok::arrow, TT_LambdaArrow);
2085	EXPECT_TOKEN(Tokens[12], tok::kw_requires, TT_RequiresClause);
2086	EXPECT_TRUE(Tokens[16]->ClosesRequiresClause);
2087	EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);
2088
2089	Tokens = annotate(Code: "[] <typename T> requires Bar<T> (T t) {}");
2090	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
2091	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2092	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2093	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2094	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
2095	// FIXME:
2096	// EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_LambdaDefinitionLParen);
2097	EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
2098
2099	Tokens = annotate(Code: "[] <typename T> requires Bar<T> (T &&t) {}");
2100	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
2101	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2102	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2103	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2104	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
2105	// FIXME:
2106	// EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_LambdaDefinitionLParen);
2107	EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_PointerOrReference);
2108	EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_LambdaLBrace);
2109
2110	Tokens = annotate(Code: "[] <typename T> requires Foo<T> || Bar<T> (T t) {}");
2111	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
2112	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2113	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2114	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2115	EXPECT_TRUE(Tokens[15]->ClosesRequiresClause);
2116	// FIXME:
2117	// EXPECT_TOKEN(Tokens[16], tok::l_paren, TT_LambdaDefinitionLParen);
2118	EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
2119
2120	Tokens = annotate(Code: "[] <typename T> requires true (T&& t) {}");
2121	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2122	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2123	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2124	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2125	EXPECT_TRUE(Tokens[7]->ClosesRequiresClause);
2126	// FIXME:
2127	// EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);
2128	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
2129	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_LambdaLBrace);
2130
2131	Tokens = annotate(Code: "[] <typename T> requires Bar<T> {}");
2132	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2133	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2134	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2135	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2136	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
2137	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_LambdaLBrace);
2138
2139	Tokens = annotate(Code: "[] <typename T> requires Bar<T> noexcept {}");
2140	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2141	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2142	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2143	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2144	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
2145	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
2146
2147	Tokens = annotate(Code: "[] <typename T> requires Bar<T> -> T {}");
2148	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2149	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2150	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2151	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2152	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
2153	EXPECT_TOKEN(Tokens[11], tok::arrow, TT_LambdaArrow);
2154	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_LambdaLBrace);
2155
2156	Tokens = annotate(Code: "[] <typename T> requires Foo<T> (T t) requires Bar<T> {}");
2157	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
2158	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2159	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2160	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
2161	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
2162	// FIXME:
2163	// EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_LambdaDefinitionLParen);
2164	EXPECT_TOKEN(Tokens[15], tok::kw_requires, TT_RequiresClause);
2165	EXPECT_TRUE(Tokens[19]->ClosesRequiresClause);
2166	EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
2167
2168	Tokens = annotate(Code: "[] <typename T = int> (T t) {}");
2169	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2170	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2171	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2172	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
2173	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);
2174	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
2175
2176	Tokens = annotate(Code: "[] <int I = 0> (T t) {}");
2177	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2178	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2179	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2180	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
2181	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);
2182	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
2183
2184	Tokens = annotate(Code: "[] <bool b = false> (T t) {}");
2185	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2186	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2187	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2188	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
2189	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_LambdaDefinitionLParen);
2190	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
2191
2192	Tokens = annotate(Code: "[] <bool b = true && false> (T&& t) {}");
2193	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
2194	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2195	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2196	EXPECT_TOKEN(Tokens[7], tok::ampamp, TT_BinaryOperator);
2197	EXPECT_TOKEN(Tokens[9], tok::greater, TT_TemplateCloser);
2198	EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_LambdaDefinitionLParen);
2199	EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_PointerOrReference);
2200	EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
2201
2202	Tokens = annotate(Code: "[] <typename T = int> requires Foo<T> (T t) {}");
2203	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
2204	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
2205	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
2206	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
2207	EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresClause);
2208	// FIXME:
2209	// EXPECT_TOKEN(Tokens[13], tok::l_paren, TT_LambdaDefinitionLParen);
2210	EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);
2211
2212	Tokens = annotate(Code: "auto foo{(std::function<int()>)[] { return 0; }};");
2213	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
2214	EXPECT_TOKEN(Tokens[13], tok::l_square, TT_LambdaLSquare);
2215	EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
2216
2217	Tokens = annotate(Code: "auto foo{(int (*)())[] { return 0; }};");
2218	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
2219	EXPECT_TOKEN(Tokens[11], tok::l_square, TT_LambdaLSquare);
2220	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_LambdaLBrace);
2221	}
2222
2223	TEST_F(TokenAnnotatorTest, UnderstandsFunctionAnnotations) {
2224	auto Tokens = annotate(Code: "template <typename T>\n"
2225	"DEPRECATED(\"Use NewClass::NewFunction instead.\")\n"
2226	"string OldFunction(const string &parameter) {}");
2227	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
2228	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_FunctionAnnotationRParen);
2229
2230	Tokens = annotate(Code: "template <typename T>\n"
2231	"A(T) noexcept;");
2232	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2233	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
2234
2235	Tokens = annotate(Code: "FOO(bar)();");
2236	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2237	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_Unknown);
2238	}
2239
2240	TEST_F(TokenAnnotatorTest, UnderstandsFunctionDeclarationNames) {
2241	auto Tokens = annotate(Code: "void f [[noreturn]] ();");
2242	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2243	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2244	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);
2245
2246	Tokens = annotate(Code: "void f [[noreturn]] () {}");
2247	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2248	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2249	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);
2250
2251	Tokens = annotate(Code: "#define FOO Foo::\n"
2252	"FOO Foo();");
2253	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2254	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_FunctionDeclarationName);
2255	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);
2256
2257	Tokens = annotate(Code: "struct Foo {\n"
2258	" Bar (*func)();\n"
2259	"};");
2260	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2261	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
2262	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionTypeLParen);
2263
2264	Tokens = annotate(Code: "void instanceof();");
2265	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2266	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2267	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
2268
2269	Tokens = annotate(Code: "#define FUNC(foo, bar, baz) \\\n"
2270	" auto foo##bar##baz() -> Type {}");
2271	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
2272	EXPECT_TOKEN(Tokens[11], tok::identifier, TT_FunctionDeclarationName);
2273	EXPECT_TOKEN(Tokens[16], tok::l_paren, TT_FunctionDeclarationLParen);
2274	EXPECT_TOKEN(Tokens[18], tok::arrow, TT_TrailingReturnArrow);
2275
2276	Tokens = annotate(Code: "void __stdcall f();");
2277	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2278	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_FunctionDeclarationName);
2279	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);
2280
2281	Tokens = annotate(Code: "int iso_time(time_t);");
2282	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2283	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2284	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
2285
2286	auto Style = getLLVMStyle();
2287	Style.TypeNames.push_back(x: "MyType");
2288	Tokens = annotate(Code: "int iso_time(MyType);", Style);
2289	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2290	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2291	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
2292	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_TypeName);
2293	}
2294
2295	TEST_F(TokenAnnotatorTest, UnderstandsCtorAndDtorDeclNames) {
2296	auto Tokens = annotate(Code: "class Foo { public: Foo(); };");
2297	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2298	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_CtorDtorDeclName);
2299	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionDeclarationLParen);
2300
2301	Tokens = annotate(Code: "class Foo { public: ~Foo(); };");
2302	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2303	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_CtorDtorDeclName);
2304	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_FunctionDeclarationLParen);
2305
2306	Tokens = annotate(Code: "struct Foo { [[deprecated]] Foo() {} };");
2307	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2308	EXPECT_TOKEN(Tokens[8], tok::identifier, TT_CtorDtorDeclName);
2309	EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_FunctionDeclarationLParen);
2310	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_FunctionLBrace);
2311
2312	Tokens = annotate(Code: "struct Foo { [[deprecated]] ~Foo() {} };");
2313	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
2314	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_CtorDtorDeclName);
2315	EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_FunctionDeclarationLParen);
2316	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);
2317
2318	Tokens = annotate(Code: "struct Foo { Foo() [[deprecated]] {} };");
2319	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2320	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
2321	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);
2322	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_FunctionLBrace);
2323
2324	Tokens = annotate(Code: "struct Foo { ~Foo() [[deprecated]] {} };");
2325	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
2326	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_CtorDtorDeclName);
2327	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_FunctionDeclarationLParen);
2328	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);
2329
2330	Tokens = annotate(Code: "struct Foo { [[deprecated]] explicit Foo() {} };");
2331	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
2332	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_CtorDtorDeclName);
2333	EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_FunctionDeclarationLParen);
2334	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);
2335
2336	Tokens = annotate(Code: "struct Foo { virtual [[deprecated]] ~Foo() {} };");
2337	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
2338	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_CtorDtorDeclName);
2339	EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);
2340	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
2341
2342	Tokens = annotate(Code: "Foo::Foo() {}");
2343	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2344	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_CtorDtorDeclName);
2345	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);
2346	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);
2347
2348	Tokens = annotate(Code: "Foo::~Foo() {}");
2349	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2350	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
2351	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);
2352	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);
2353
2354	Tokens = annotate(Code: "struct Test {\n"
2355	" Test()\n"
2356	" : l([] {\n"
2357	" Short::foo();\n"
2358	" }) {}\n"
2359	"};");
2360	ASSERT_EQ(Tokens.size(), 25u) << Tokens;
2361	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
2362	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);
2363	EXPECT_TOKEN(Tokens[14], tok::identifier, TT_Unknown);
2364	}
2365
2366	TEST_F(TokenAnnotatorTest, UnderstandsC11GenericSelection) {
2367	auto Tokens = annotate(Code: "_Generic(x, int: 1, default: 0)");
2368	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2369	EXPECT_TOKEN(Tokens[0], tok::kw__Generic, TT_Unknown);
2370	EXPECT_TOKEN(Tokens[5], tok::colon, TT_GenericSelectionColon);
2371	EXPECT_TOKEN(Tokens[9], tok::colon, TT_GenericSelectionColon);
2372	}
2373
2374	TEST_F(TokenAnnotatorTest, UnderstandsTrailingReturnArrow) {
2375	auto Tokens = annotate(Code: "auto f() -> int;");
2376	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2377	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
2378
2379	Tokens = annotate(Code: "auto operator->() -> int;");
2380	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2381	EXPECT_TOKEN(Tokens[2], tok::arrow, TT_OverloadedOperator);
2382	EXPECT_TOKEN(Tokens[5], tok::arrow, TT_TrailingReturnArrow);
2383
2384	Tokens = annotate(Code: "auto operator++(int) -> int;");
2385	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2386	EXPECT_TOKEN(Tokens[6], tok::arrow, TT_TrailingReturnArrow);
2387
2388	Tokens = annotate(Code: "auto operator=() -> int;");
2389	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2390	EXPECT_TOKEN(Tokens[5], tok::arrow, TT_TrailingReturnArrow);
2391
2392	Tokens = annotate(Code: "auto operator=(int) -> int;");
2393	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2394	EXPECT_TOKEN(Tokens[6], tok::arrow, TT_TrailingReturnArrow);
2395
2396	Tokens = annotate(Code: "auto foo() -> auto { return Val; }");
2397	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2398	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
2399
2400	Tokens = annotate(Code: "struct S { auto bar() const -> int; };");
2401	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2402	EXPECT_TOKEN(Tokens[8], tok::arrow, TT_TrailingReturnArrow);
2403
2404	// Not trailing return arrows
2405	Tokens = annotate(Code: "auto a = b->c;");
2406	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2407	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);
2408
2409	Tokens = annotate(Code: "auto a = (b)->c;");
2410	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2411	EXPECT_TOKEN(Tokens[6], tok::arrow, TT_Unknown);
2412
2413	Tokens = annotate(Code: "auto a = b()->c;");
2414	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2415	EXPECT_TOKEN(Tokens[6], tok::arrow, TT_Unknown);
2416
2417	Tokens = annotate(Code: "auto a = b->c();");
2418	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2419	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);
2420
2421	Tokens = annotate(Code: "decltype(auto) a = b()->c;");
2422	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2423	EXPECT_TOKEN(Tokens[9], tok::arrow, TT_Unknown);
2424
2425	Tokens = annotate(Code: "void f() { auto a = b->c(); }");
2426	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2427	EXPECT_TOKEN(Tokens[9], tok::arrow, TT_Unknown);
2428
2429	Tokens = annotate(Code: "void f() { auto a = b()->c; }");
2430	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2431	EXPECT_TOKEN(Tokens[11], tok::arrow, TT_Unknown);
2432
2433	Tokens = annotate(Code: "#define P(ptr) auto p = (ptr)->p");
2434	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2435	EXPECT_TOKEN(Tokens[12], tok::arrow, TT_Unknown);
2436
2437	Tokens = annotate(Code: "void f() FOO(foo->bar);");
2438	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2439	EXPECT_TOKEN(Tokens[7], tok::arrow, TT_Unknown);
2440
2441	Tokens = annotate(Code: "__attribute__((cold)) C() : Base(obj->func()) {}");
2442	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
2443	EXPECT_TOKEN(Tokens[13], tok::arrow, TT_Unknown);
2444
2445	Tokens = annotate(Code: "Class<Type>{foo}->func(arg);");
2446	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2447	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_Unknown); // Not FunctionLBrace
2448	EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);
2449	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
2450	EXPECT_TOKEN(Tokens[7], tok::arrow, TT_Unknown);
2451
2452	auto Style = getLLVMStyle();
2453	Style.StatementAttributeLikeMacros.push_back(x: "emit");
2454	Tokens = annotate(Code: "emit foo()->bar;", Style);
2455	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2456	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_StatementAttributeLikeMacro);
2457	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);
2458
2459	// Mixed
2460	Tokens = annotate(Code: "auto f() -> int { auto a = b()->c; }");
2461	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
2462	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
2463	EXPECT_TOKEN(Tokens[13], tok::arrow, TT_Unknown);
2464	}
2465
2466	TEST_F(TokenAnnotatorTest, UnderstandHashInMacro) {
2467	auto Tokens = annotate(Code: "#define Foo(Bar) \\\n"
2468	" { \\\n"
2469	" #Bar \\\n"
2470	" }");
2471	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2472	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
2473	EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
2474
2475	Tokens = annotate(Code: "#define Foo(Bar) \\\n"
2476	" { #Bar }");
2477	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2478	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
2479	EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
2480
2481	Tokens = annotate(Code: "#define FOO(typeName, realClass) \\\n"
2482	" {#typeName, foo<Foo>(new foo<realClass>(#typeName))}");
2483	ASSERT_EQ(Tokens.size(), 29u) << Tokens;
2484	EXPECT_BRACE_KIND(Tokens[8], BK_BracedInit);
2485	EXPECT_BRACE_KIND(Tokens[27], BK_BracedInit);
2486	}
2487
2488	TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacros) {
2489	// '__attribute__' has special handling.
2490	auto Tokens = annotate(Code: "__attribute__((X)) void Foo(void);");
2491	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2492	EXPECT_TOKEN(Tokens[0], tok::kw___attribute, TT_Unknown);
2493	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
2494	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);
2495
2496	// Generic macro has no special handling in this location.
2497	Tokens = annotate(Code: "A(X) void Foo(void);");
2498	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2499	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2500	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_Unknown);
2501
2502	// Add a custom AttributeMacro. Test that it has the same behavior.
2503	FormatStyle Style = getLLVMStyle();
2504	Style.AttributeMacros.push_back(x: "A");
2505
2506	// An "AttributeMacro" gets annotated like '__attribute__'.
2507	Tokens = annotate(Code: "A(X) void Foo(void);", Style);
2508	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2509	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_AttributeMacro);
2510	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
2511	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);
2512	}
2513
2514	TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCDecl) {
2515	// '__attribute__' has special handling.
2516	auto Tokens = annotate(Code: "__attribute__((X)) @interface Foo");
2517	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2518	EXPECT_TOKEN(Tokens[0], tok::kw___attribute, TT_Unknown);
2519	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
2520	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);
2521
2522	// Generic macro has no special handling in this location.
2523	Tokens = annotate(Code: "A(X) @interface Foo");
2524	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2525	// Note: Don't check token-type as a random token in this position is hard to
2526	// reason about.
2527	EXPECT_TOKEN_KIND(Tokens[0], tok::identifier);
2528	EXPECT_TOKEN_KIND(Tokens[1], tok::l_paren);
2529
2530	// Add a custom AttributeMacro. Test that it has the same behavior.
2531	FormatStyle Style = getLLVMStyle();
2532	Style.AttributeMacros.push_back(x: "A");
2533
2534	// An "AttributeMacro" gets annotated like '__attribute__'.
2535	Tokens = annotate(Code: "A(X) @interface Foo", Style);
2536	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2537	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_AttributeMacro);
2538	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
2539	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);
2540	}
2541
2542	TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCMethodDecl) {
2543	// '__attribute__' has special handling.
2544	auto Tokens = annotate(Code: "- (id)init __attribute__((X));");
2545	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2546	EXPECT_TOKEN(Tokens[5], tok::kw___attribute, TT_Unknown);
2547	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_AttributeLParen);
2548	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_AttributeRParen);
2549
2550	// Generic macro has no special handling in this location.
2551	Tokens = annotate(Code: "- (id)init A(X);");
2552	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2553	// Note: Don't check token-type as a random token in this position is hard to
2554	// reason about.
2555	EXPECT_TOKEN_KIND(Tokens[5], tok::identifier);
2556	EXPECT_TOKEN_KIND(Tokens[6], tok::l_paren);
2557
2558	// Add a custom AttributeMacro. Test that it has the same behavior.
2559	FormatStyle Style = getLLVMStyle();
2560	Style.AttributeMacros.push_back(x: "A");
2561
2562	// An "AttributeMacro" gets annotated like '__attribute__'.
2563	Tokens = annotate(Code: "- (id)init A(X);", Style);
2564	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2565	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_AttributeMacro);
2566	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_AttributeLParen);
2567	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_AttributeRParen);
2568	}
2569
2570	TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCProperty) {
2571	// '__attribute__' has special handling.
2572	auto Tokens = annotate(Code: "@property(weak) id delegate __attribute__((X));");
2573	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2574	EXPECT_TOKEN(Tokens[7], tok::kw___attribute, TT_Unknown);
2575	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_AttributeLParen);
2576	EXPECT_TOKEN(Tokens[12], tok::r_paren, TT_AttributeRParen);
2577
2578	// Generic macro has no special handling in this location.
2579	Tokens = annotate(Code: "@property(weak) id delegate A(X);");
2580	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2581	// Note: Don't check token-type as a random token in this position is hard to
2582	// reason about.
2583	EXPECT_TOKEN_KIND(Tokens[7], tok::identifier);
2584	EXPECT_TOKEN_KIND(Tokens[8], tok::l_paren);
2585
2586	// Add a custom AttributeMacro. Test that it has the same behavior.
2587	FormatStyle Style = getLLVMStyle();
2588	Style.AttributeMacros.push_back(x: "A");
2589
2590	// An "AttributeMacro" gets annotated like '__attribute__'.
2591	Tokens = annotate(Code: "@property(weak) id delegate A(X);", Style);
2592	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2593	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_AttributeMacro);
2594	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_AttributeLParen);
2595	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_AttributeRParen);
2596	}
2597
2598	TEST_F(TokenAnnotatorTest, UnderstandsVerilogOperators) {
2599	auto Annotate = [this](StringRef Code) {
2600	return annotate(Code, Style: getLLVMStyle(Language: FormatStyle::LK_Verilog));
2601	};
2602	// Test that unary operators get labeled as such and that operators like '++'
2603	// don't get split.
2604	tok::TokenKind Unary[] = {tok::plus, tok::minus, tok::exclaim,
2605	tok::tilde, tok::amp, tok::pipe,
2606	tok::caret, tok::plusplus, tok::minusminus};
2607	for (auto Kind : Unary) {
2608	auto Tokens =
2609	Annotate(std::string("x = ") + tok::getPunctuatorSpelling(Kind) + "x;");
2610	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2611	EXPECT_TOKEN(Tokens[2], Kind, TT_UnaryOperator);
2612	}
2613	// Operators formed by joining two operators like '^~'. For some of these
2614	// joined operators, we don't have a separate type, so we only test for their
2615	// precedence.
2616	std::pair<prec::Level, std::string> JoinedBinary[] = {
2617	{prec::Comma, "->"}, {prec::Comma, "<->"},
2618	{prec::Assignment, "+="}, {prec::Assignment, "-="},
2619	{prec::Assignment, "*="}, {prec::Assignment, "/="},
2620	{prec::Assignment, "%="}, {prec::Assignment, "&="},
2621	{prec::Assignment, "^="}, {prec::Assignment, "<<="},
2622	{prec::Assignment, ">>="}, {prec::Assignment, "<<<="},
2623	{prec::Assignment, ">>>="}, {prec::LogicalOr, "||"},
2624	{prec::LogicalAnd, "&&"}, {prec::Equality, "=="},
2625	{prec::Equality, "!="}, {prec::Equality, "==="},
2626	{prec::Equality, "!=="}, {prec::Equality, "==?"},
2627	{prec::Equality, "!=?"}, {prec::ExclusiveOr, "~^"},
2628	{prec::ExclusiveOr, "^~"},
2629	};
2630	for (auto Operator : JoinedBinary) {
2631	auto Tokens = Annotate(std::string("x = x ") + Operator.second + " x;");
2632	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2633	EXPECT_TOKEN_TYPE(Tokens[3], TT_BinaryOperator);
2634	EXPECT_TOKEN_PRECEDENCE(Tokens[3], Operator.first);
2635	}
2636	// '~^' and '^~' can be unary as well as binary operators.
2637	auto Tokens = Annotate("x = ~^x;");
2638	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2639	EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator);
2640	Tokens = Annotate("x = ^~x;");
2641	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2642	EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator);
2643	// The unary operators '~&' and '~|' can only be unary operators. The current
2644	// implementation treats each of them as separate unary '~' and '&' or '|'
2645	// operators, which is enough for formatting purposes. In FormatTestVerilog,
2646	// there is a test that there is no space in between. And even if a new line
2647	// is inserted between the '~' and '|', the semantic meaning is the same as
2648	// the joined operator, so the CanBreakBefore property doesn't need to be
2649	// false for the second operator.
2650	Tokens = Annotate("x = ~&x;");
2651	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2652	EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator);
2653	EXPECT_TOKEN(Tokens[3], tok::amp, TT_UnaryOperator);
2654	Tokens = Annotate("x = ~|x;");
2655	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2656	EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator);
2657	EXPECT_TOKEN(Tokens[3], tok::pipe, TT_UnaryOperator);
2658	// Test for block label colons.
2659	Tokens = Annotate("begin : x\n"
2660	"end : x");
2661	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2662	EXPECT_TOKEN(Tokens[1], tok::colon, TT_VerilogBlockLabelColon);
2663	EXPECT_TOKEN(Tokens[4], tok::colon, TT_VerilogBlockLabelColon);
2664	// Test that the dimension colon is annotated correctly.
2665	Tokens = Annotate("var [1 : 0] x;");
2666	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2667	EXPECT_TOKEN(Tokens[3], tok::colon, TT_BitFieldColon);
2668	Tokens = Annotate("extern function [1 : 0] x;");
2669	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2670	EXPECT_TOKEN(Tokens[4], tok::colon, TT_BitFieldColon);
2671	Tokens = Annotate("module test\n"
2672	" (input wire [7 : 0] a[7 : 0]);\n"
2673	"endmodule");
2674	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
2675	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_VerilogDimensionedTypeName);
2676	EXPECT_TOKEN(Tokens[7], tok::colon, TT_BitFieldColon);
2677	EXPECT_TOKEN(Tokens[13], tok::colon, TT_BitFieldColon);
2678	// Test case labels and ternary operators.
2679	Tokens = Annotate("case (x)\n"
2680	" x:\n"
2681	" x;\n"
2682	"endcase");
2683	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2684	EXPECT_TOKEN(Tokens[5], tok::colon, TT_CaseLabelColon);
2685	Tokens = Annotate("case (x)\n"
2686	" x ? x : x:\n"
2687	" x;\n"
2688	"endcase");
2689	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2690	EXPECT_TOKEN(Tokens[5], tok::question, TT_ConditionalExpr);
2691	EXPECT_TOKEN(Tokens[7], tok::colon, TT_ConditionalExpr);
2692	EXPECT_TOKEN(Tokens[9], tok::colon, TT_CaseLabelColon);
2693	// Non-blocking assignments.
2694	Tokens = Annotate("a <= b;");
2695	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
2696	EXPECT_TOKEN(Tokens[1], tok::lessequal, TT_BinaryOperator);
2697	EXPECT_TOKEN_PRECEDENCE(Tokens[1], prec::Assignment);
2698	Tokens = Annotate("if (a <= b) break;");
2699	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2700	EXPECT_TOKEN(Tokens[3], tok::lessequal, TT_BinaryOperator);
2701	EXPECT_TOKEN_PRECEDENCE(Tokens[3], prec::Relational);
2702	Tokens = Annotate("a <= b <= a;");
2703	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2704	EXPECT_TOKEN(Tokens[1], tok::lessequal, TT_BinaryOperator);
2705	EXPECT_TOKEN_PRECEDENCE(Tokens[1], prec::Assignment);
2706	EXPECT_TOKEN(Tokens[3], tok::lessequal, TT_BinaryOperator);
2707	EXPECT_TOKEN_PRECEDENCE(Tokens[3], prec::Relational);
2708
2709	// Port lists in module instantiation.
2710	Tokens = Annotate("module_x instance_1(port_1), instance_2(port_2);");
2711	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2712	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_VerilogInstancePortLParen);
2713	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_VerilogInstancePortLParen);
2714	Tokens = Annotate("module_x #(parameter) instance_1(port_1), "
2715	"instance_2(port_2);");
2716	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2717	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_VerilogInstancePortLParen);
2718	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_VerilogInstancePortLParen);
2719	EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_VerilogInstancePortLParen);
2720
2721	// Condition parentheses.
2722	Tokens = Annotate("assert (x);");
2723	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2724	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2725	Tokens = Annotate("assert #0 (x);");
2726	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2727	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_ConditionLParen);
2728	Tokens = Annotate("assert final (x);");
2729	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2730	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);
2731	Tokens = Annotate("foreach (x[x]) continue;");
2732	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2733	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2734	Tokens = Annotate("repeat (x[x]) continue;");
2735	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2736	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2737	Tokens = Annotate("case (x) endcase;");
2738	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2739	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2740
2741	// Sensitivity list. The TT_Unknown type is clearly not binding for the
2742	// future, please adapt if those tokens get annotated. This test is only here
2743	// to prevent the comma from being annotated as TT_VerilogInstancePortComma.
2744	Tokens = Annotate("always @(posedge x, posedge y);");
2745	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2746	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_Unknown);
2747	EXPECT_TOKEN(Tokens[5], tok::comma, TT_Unknown);
2748	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
2749
2750	// String literals in concatenation.
2751	Tokens = Annotate("x = {\"\"};");
2752	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2753	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_StringInConcatenation);
2754	Tokens = Annotate("x = {\"\", \"\"};");
2755	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2756	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_StringInConcatenation);
2757	EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_StringInConcatenation);
2758	Tokens = Annotate("x = '{{\"\"}};");
2759	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2760	EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_StringInConcatenation);
2761	// Cases where the string should not be annotated that type. Fix the
2762	// `TT_Unknown` if needed in the future.
2763	Tokens = Annotate("x = {\"\" == \"\"};");
2764	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2765	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_Unknown);
2766	EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_Unknown);
2767	Tokens = Annotate("x = {(\"\")};");
2768	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2769	EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_Unknown);
2770	Tokens = Annotate("x = '{\"\"};");
2771	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2772	EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_Unknown);
2773
2774	// Module headers.
2775	Tokens = Annotate("module x();\n"
2776	"endmodule");
2777	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2778	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_VerilogMultiLineListLParen);
2779	Tokens = Annotate("function automatic `x x();\n"
2780	"endmodule");
2781	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2782	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_VerilogMultiLineListLParen);
2783	Tokens = Annotate("function automatic x``x x();\n"
2784	"endmodule");
2785	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2786	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_VerilogMultiLineListLParen);
2787	Tokens = Annotate("function automatic x::x x();\n"
2788	"endmodule");
2789	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2790	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_VerilogMultiLineListLParen);
2791
2792	// Escaped identifiers.
2793	Tokens = Annotate(R"(\busa+index)");
2794	ASSERT_EQ(Tokens.size(), 2u) << Tokens;
2795	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2796	Tokens = Annotate(R"(\busa+index ;)");
2797	ASSERT_EQ(Tokens.size(), 3u) << Tokens;
2798	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2799	EXPECT_EQ(Tokens[0]->TokenText, R"(\busa+index)");
2800	EXPECT_TOKEN(Tokens[1], tok::semi, TT_Unknown);
2801	Tokens = Annotate(R"(\busa+index
2802	;)");
2803	ASSERT_EQ(Tokens.size(), 3u) << Tokens;
2804	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2805	EXPECT_TOKEN(Tokens[1], tok::semi, TT_Unknown);
2806	// The escaped identifier can be broken by an escaped newline. The result is
2807	// still 1 identifier.
2808	Tokens = Annotate(R"(\busa+index\
2809	+
2810	;)");
2811	ASSERT_EQ(Tokens.size(), 3u) << Tokens;
2812	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2813	EXPECT_EQ(Tokens[0]->TokenText, R"(\busa+index\
2814	+)");
2815	EXPECT_TOKEN(Tokens[1], tok::semi, TT_Unknown);
2816	// An escaped newline should not be treated as an escaped identifier.
2817	Tokens = Annotate("\\\n");
2818	ASSERT_EQ(Tokens.size(), 1u) << Tokens;
2819	EXPECT_TOKEN(Tokens[0], tok::eof, TT_Unknown);
2820	// Macros.
2821	Tokens = Annotate("`define x x``x");
2822	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2823	EXPECT_TOKEN(Tokens[0], tok::hash, TT_Unknown);
2824	EXPECT_TOKEN(Tokens[4], tok::hashhash, TT_Unknown);
2825	}
2826
2827	TEST_F(TokenAnnotatorTest, UnderstandTableGenTokens) {
2828	auto Style = getLLVMStyle(Language: FormatStyle::LK_TableGen);
2829	ASSERT_TRUE(Style.isTableGen());
2830
2831	TestLexer Lexer(Allocator, Buffers, Style);
2832	AdditionalKeywords Keywords(Lexer.IdentTable);
2833	auto Annotate = [&Lexer](StringRef Code) { return Lexer.annotate(Code); };
2834
2835	// Additional keywords representation test.
2836	auto Tokens = Annotate("def foo : Bar<1>;");
2837	ASSERT_TRUE(Keywords.isTableGenKeyword(*Tokens[0]));
2838	ASSERT_TRUE(Keywords.isTableGenDefinition(*Tokens[0]));
2839	ASSERT_TRUE(Tokens[0]->is(Keywords.kw_def));
2840	ASSERT_TRUE(Tokens[1]->is(TT_StartOfName));
2841
2842	// Code, the multiline string token.
2843	Tokens = Annotate("[{ code is multiline string }]");
2844	ASSERT_EQ(Tokens.size(), 2u) << Tokens;
2845	EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString);
2846	EXPECT_FALSE(Tokens[0]->IsMultiline);
2847	// Case with multiple lines.
2848	Tokens = Annotate("[{ It can break\n"
2849	" across lines and the line breaks\n"
2850	" are retained in \n"
2851	" the string. }]");
2852	ASSERT_EQ(Tokens.size(), 2u) << Tokens;
2853	EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString);
2854	EXPECT_EQ(Tokens[0]->ColumnWidth, sizeof("[{ It can break\n") - 1);
2855	EXPECT_TRUE(Tokens[0]->IsMultiline);
2856	EXPECT_EQ(Tokens[0]->LastLineColumnWidth, sizeof(" the string. }]") - 1);
2857
2858	// Numeric literals.
2859	Tokens = Annotate("1234");
2860	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2861	Tokens = Annotate("-1");
2862	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2863	Tokens = Annotate("+1234");
2864	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2865	Tokens = Annotate("0b0110");
2866	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2867	Tokens = Annotate("0x1abC");
2868	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2869
2870	// Identifier tokens. In TableGen, identifiers can begin with a number.
2871	// In ambiguous cases, the lexer tries to lex it as a number.
2872	// Even if the try fails, it does not fall back to identifier lexing and
2873	// regard as an error.
2874	// The ambiguity is not documented. The result of those tests are based on the
2875	// implementation of llvm::TGLexer::LexToken.
2876	// This is invalid syntax of number, but not an identifier.
2877	Tokens = Annotate("0x1234x");
2878	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2879	Tokens = Annotate("identifier");
2880	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2881	// Identifier beginning with a number.
2882	Tokens = Annotate("0x");
2883	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2884	Tokens = Annotate("2dVector");
2885	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2886	Tokens = Annotate("01234Vector");
2887	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2888
2889	// Structured statements.
2890	Tokens = Annotate("class Foo {}");
2891	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
2892	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_FunctionLBrace);
2893	Tokens = Annotate("def Def: Foo {}");
2894	EXPECT_TOKEN(Tokens[2], tok::colon, TT_InheritanceColon);
2895	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
2896	Tokens = Annotate("if cond then {} else {}");
2897	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_ControlStatementLBrace);
2898	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_ElseLBrace);
2899	Tokens = Annotate("defset Foo Def2 = {}");
2900	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
2901
2902	// Bang Operators.
2903	Tokens = Annotate("!foreach");
2904	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenBangOperator);
2905	Tokens = Annotate("!if");
2906	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenBangOperator);
2907	Tokens = Annotate("!cond");
2908	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenCondOperator);
2909
2910	// The paste operator should not be treated as a preprocessor directive even
2911	// if it is on a separate line.
2912	Tokens = Annotate("def x\n"
2913	"#embed {}");
2914	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2915	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
2916	EXPECT_EQ(Tokens[1]->Next, Tokens[2]);
2917	Tokens = Annotate("def x\n"
2918	"#define x\n"
2919	"#embed {}");
2920	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2921	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
2922	EXPECT_EQ(Tokens[1]->Next, Tokens[5]);
2923	Tokens = Annotate("def x\n"
2924	"#ifdef x\n"
2925	"#else\n"
2926	"#endif\n"
2927	"#embed {}");
2928	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2929	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
2930	EXPECT_EQ(Tokens[1]->Next, Tokens[9]);
2931
2932	auto AnnotateValue = [this, &Style](StringRef Code) {
2933	// Values are annotated only in specific context.
2934	auto Result = annotate(Code: ("def X { let V = " + Code + "; }").str(), Style);
2935	return decltype(Result){Result.begin() + 6, Result.end() - 3};
2936	};
2937	// Both of bang/cond operators.
2938	Tokens = AnnotateValue("!cond(!eq(x, 0): 1, true: x)");
2939	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2940	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenCondOperator);
2941	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TableGenBangOperator);
2942	EXPECT_TOKEN(Tokens[8], tok::colon, TT_TableGenCondOperatorColon);
2943	EXPECT_TOKEN(Tokens[10], tok::comma, TT_TableGenCondOperatorComma);
2944	EXPECT_TOKEN(Tokens[12], tok::colon, TT_TableGenCondOperatorColon);
2945	// DAGArg values with operator identifier
2946	Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
2947	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2948	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpener);
2949	EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColon);
2950	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown); // $src1
2951	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListComma);
2952	EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColon);
2953	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2954	// List literal
2955	Tokens = AnnotateValue("[1, 2, 3]");
2956	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2957	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_TableGenListOpener);
2958	EXPECT_TOKEN(Tokens[6], tok::r_square, TT_TableGenListCloser);
2959	// Suffixes of values
2960	Tokens = AnnotateValue("valid.field");
2961	ASSERT_EQ(Tokens.size(), 3u) << Tokens;
2962	EXPECT_TOKEN(Tokens[1], tok::period, TT_TableGenValueSuffix);
2963	// Code
2964	Tokens = AnnotateValue("[{ code is multiline string }]");
2965	ASSERT_EQ(Tokens.size(), 1u) << Tokens;
2966	EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString);
2967
2968	// The definition
2969	Tokens = annotate(Code: "def Def : Parent<Child> {}", Style);
2970	ASSERT_EQ(Tokens.size(), 10u) << Tokens; // This contains eof.
2971	// We use inheritance colon and function brace. They are enough.
2972	EXPECT_TOKEN(Tokens[2], tok::colon, TT_InheritanceColon);
2973	EXPECT_TOKEN(Tokens[4], tok::less, TT_TemplateOpener);
2974	EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
2975	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
2976
2977	// DAGArg breaking options. They use different token types depending on what
2978	// is specified.
2979	Style.TableGenBreakInsideDAGArg = FormatStyle::DAS_BreakElements;
2980
2981	// When TableGenBreakInsideDAGArg is DAS_BreakElements and
2982	// TableGenBreakingDAGArgOperators is not specified, it makes all the DAGArg
2983	// elements to have line break.
2984	Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
2985	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2986	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
2987	EXPECT_TOKEN(Tokens[1], tok::identifier,
2988	TT_TableGenDAGArgOperatorID); // ins
2989	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
2990	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2991
2992	Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
2993	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2994	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
2995	EXPECT_TOKEN(Tokens[1], tok::identifier,
2996	TT_TableGenDAGArgOperatorID); // other
2997	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
2998	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2999
3000	// For non-identifier operators, breaks after the operator.
3001	Tokens = AnnotateValue("(!cast<Type>(\"Name\") type1:$src1, type2:$src2)");
3002	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
3003	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
3004	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_TableGenDAGArgOperatorToBreak);
3005	EXPECT_TOKEN(Tokens[11], tok::comma, TT_TableGenDAGArgListCommaToBreak);
3006	EXPECT_TOKEN(Tokens[15], tok::r_paren, TT_TableGenDAGArgCloser);
3007
3008	Style.TableGenBreakInsideDAGArg = FormatStyle::DAS_BreakAll;
3009
3010	// When TableGenBreakInsideDAGArg is DAS_BreakAll and
3011	// TableGenBreakingDAGArgOperators is not specified, it makes all the DAGArg
3012	// to have line break inside it.
3013	Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
3014	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3015	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
3016	EXPECT_TOKEN(Tokens[1], tok::identifier,
3017	TT_TableGenDAGArgOperatorToBreak); // ins
3018	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
3019	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
3020
3021	Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
3022	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3023	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
3024	EXPECT_TOKEN(Tokens[1], tok::identifier,
3025	TT_TableGenDAGArgOperatorToBreak); // other
3026	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
3027	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
3028
3029	// If TableGenBreakingDAGArgOperators is specified, it is limited to the
3030	// specified operators.
3031	Style.TableGenBreakingDAGArgOperators = {"ins", "outs"};
3032	Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
3033	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3034	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
3035	EXPECT_TOKEN(Tokens[1], tok::identifier,
3036	TT_TableGenDAGArgOperatorToBreak); // ins
3037	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
3038	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
3039
3040	Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
3041	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3042	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpener);
3043	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other
3044	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListComma);
3045	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
3046
3047	// If TableGenBreakingDAGArgOperators is enabled, it uses
3048	// TT_TableGenDAGArgListColonToAlign to annotate the colon to align.
3049	Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled = true;
3050	Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
3051	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3052	EXPECT_TOKEN(Tokens[1], tok::identifier,
3053	TT_TableGenDAGArgOperatorToBreak); // ins
3054	EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColonToAlign);
3055	EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColonToAlign);
3056
3057	Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
3058	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3059	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other
3060	EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColon);
3061	EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColon);
3062	}
3063
3064	TEST_F(TokenAnnotatorTest, UnderstandConstructors) {
3065	auto Tokens = annotate(Code: "Class::Class() : BaseClass(), Member() {}");
3066
3067	// The TT_Unknown is clearly not binding for the future, please adapt if those
3068	// tokens get annotated.
3069	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
3070	EXPECT_TOKEN(Tokens[5], tok::colon, TT_CtorInitializerColon);
3071	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
3072	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_Unknown);
3073	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
3074	EXPECT_TOKEN(Tokens[9], tok::comma, TT_CtorInitializerComma);
3075	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_Unknown);
3076	EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_Unknown);
3077	EXPECT_TOKEN(Tokens[12], tok::r_paren, TT_Unknown);
3078	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
3079	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3080
3081	Tokens = annotate(Code: "Class::Class() : BaseClass{}, Member{} {}");
3082	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
3083	EXPECT_TOKEN(Tokens[5], tok::colon, TT_CtorInitializerColon);
3084	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
3085	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_Unknown);
3086	EXPECT_TOKEN(Tokens[8], tok::r_brace, TT_Unknown);
3087	EXPECT_TOKEN(Tokens[9], tok::comma, TT_CtorInitializerComma);
3088	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_Unknown);
3089	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_Unknown);
3090	EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_Unknown);
3091	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
3092	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3093
3094	Tokens = annotate(Code: "class Class {\n"
3095	" Class() : BaseClass() {\n"
3096	"#if 0\n"
3097	" // comment\n"
3098	"#endif\n"
3099	" }\n"
3100	" Class f();\n"
3101	"}");
3102	ASSERT_EQ(Tokens.size(), 25u) << Tokens;
3103	EXPECT_TOKEN(Tokens[6], tok::colon, TT_CtorInitializerColon);
3104	EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_FunctionLBrace);
3105	EXPECT_BRACE_KIND(Tokens[10], BK_Block);
3106	}
3107
3108	TEST_F(TokenAnnotatorTest, UnderstandsConditionParens) {
3109	auto Tokens = annotate(Code: "if (x) {}");
3110	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3111	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
3112	Tokens = annotate(Code: "if constexpr (x) {}");
3113	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3114	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);
3115	Tokens = annotate(Code: "if CONSTEXPR (x) {}");
3116	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3117	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);
3118	Tokens = annotate(Code: "if (x) {} else if (x) {}");
3119	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3120	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
3121	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_ConditionLParen);
3122	}
3123
3124	TEST_F(TokenAnnotatorTest, CSharpNullableTypes) {
3125	FormatStyle Style = getGoogleStyle(Language: FormatStyle::LK_CSharp);
3126
3127	auto Tokens = annotate(Code: "int? a;", Style);
3128	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
3129	EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
3130
3131	Tokens = annotate(Code: "int? a = 1;", Style);
3132	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3133	EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
3134
3135	Tokens = annotate(Code: "int?)", Style);
3136	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
3137	EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
3138
3139	Tokens = annotate(Code: "int?>", Style);
3140	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
3141	EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
3142
3143	Tokens = annotate(Code: "{\n"
3144	" int? a;\n"
3145	" if (b is int?)\n"
3146	" f();\n"
3147	" var foo = A<Foo?>();\n"
3148	"}",
3149	Style);
3150	ASSERT_EQ(Tokens.size(), 29u) << Tokens;
3151	EXPECT_TOKEN(Tokens[2], tok::question, TT_CSharpNullable);
3152	EXPECT_TOKEN(Tokens[10], tok::question, TT_CSharpNullable);
3153	EXPECT_TOKEN(Tokens[20], tok::less, TT_TemplateOpener);
3154	EXPECT_TOKEN(Tokens[22], tok::question, TT_CSharpNullable);
3155
3156	Tokens = annotate(Code: "cond? id : id2", Style);
3157	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3158	EXPECT_TOKEN(Tokens[1], tok::question, TT_ConditionalExpr);
3159
3160	Tokens = annotate(Code: "cond ? cond2 ? : id1 : id2", Style);
3161	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3162	EXPECT_TOKEN(Tokens[1], tok::question, TT_ConditionalExpr);
3163	}
3164
3165	TEST_F(TokenAnnotatorTest, CSharpGenericTypeConstraint) {
3166	auto Tokens = annotate(Code: "namespace A {\n"
3167	" delegate T MyDelegate<T>()\n"
3168	" where T : new();\n"
3169	"}",
3170	Style: getGoogleStyle(Language: FormatStyle::LK_CSharp));
3171	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
3172	EXPECT_TOKEN(Tokens[18], tok::r_brace, TT_NamespaceRBrace);
3173	}
3174
3175	TEST_F(TokenAnnotatorTest, CSharpNewModifier) {
3176	auto Tokens = annotate(Code: "new public class NestedC {\n"
3177	" public int x = 100;\n"
3178	"}",
3179	Style: getGoogleStyle(Language: FormatStyle::LK_CSharp));
3180	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
3181	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_ClassHeadName);
3182	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ClassLBrace);
3183	EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_ClassRBrace);
3184	}
3185
3186	TEST_F(TokenAnnotatorTest, UnderstandsLabels) {
3187	auto Tokens = annotate(Code: "{ x: break; }");
3188	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3189	EXPECT_TOKEN(Tokens[2], tok::colon, TT_GotoLabelColon);
3190
3191	Tokens = annotate(Code: "{ case x: break; }");
3192	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3193	EXPECT_TOKEN(Tokens[3], tok::colon, TT_CaseLabelColon);
3194
3195	Tokens = annotate(Code: "{ x: { break; } }");
3196	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3197	EXPECT_TOKEN(Tokens[2], tok::colon, TT_GotoLabelColon);
3198
3199	Tokens = annotate(Code: "{ case x: { break; } }");
3200	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3201	EXPECT_TOKEN(Tokens[3], tok::colon, TT_CaseLabelColon);
3202
3203	Tokens = annotate(Code: "#define FOO label:");
3204	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3205	EXPECT_TOKEN(Tokens[4], tok::colon, TT_GotoLabelColon);
3206
3207	Tokens = annotate(Code: "#define FOO \\\n"
3208	"label: \\\n"
3209	" break;");
3210	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3211	EXPECT_TOKEN(Tokens[4], tok::colon, TT_GotoLabelColon);
3212	}
3213
3214	TEST_F(TokenAnnotatorTest, UnderstandsNestedBlocks) {
3215	// The closing braces are not annotated. It doesn't seem to cause a problem.
3216	// So we only test for the opening braces.
3217	auto Tokens = annotate(Code: "{\n"
3218	" {\n"
3219	" { int a = 0; }\n"
3220	" }\n"
3221	" {}\n"
3222	"}");
3223	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3224	EXPECT_BRACE_KIND(Tokens[0], BK_Block);
3225	EXPECT_BRACE_KIND(Tokens[1], BK_Block);
3226	EXPECT_BRACE_KIND(Tokens[2], BK_Block);
3227	EXPECT_BRACE_KIND(Tokens[10], BK_Block);
3228	}
3229
3230	TEST_F(TokenAnnotatorTest, UnderstandDesignatedInitializers) {
3231	auto Tokens = annotate(Code: "SomeStruct { .a = 1 };");
3232	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3233	EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
3234	EXPECT_TOKEN(Tokens[2], tok::period, TT_DesignatedInitializerPeriod);
3235
3236	Tokens = annotate(Code: "SomeStruct { .a = 1, .b = 2 };");
3237	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3238	EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
3239	EXPECT_TOKEN(Tokens[2], tok::period, TT_DesignatedInitializerPeriod);
3240	EXPECT_TOKEN(Tokens[7], tok::period, TT_DesignatedInitializerPeriod);
3241
3242	Tokens = annotate(Code: "SomeStruct {\n"
3243	"#ifdef FOO\n"
3244	" .a = 1,\n"
3245	"#endif\n"
3246	" .b = 2\n"
3247	"};");
3248	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
3249	EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
3250	EXPECT_TOKEN(Tokens[5], tok::period, TT_DesignatedInitializerPeriod);
3251	EXPECT_TOKEN(Tokens[12], tok::period, TT_DesignatedInitializerPeriod);
3252
3253	Tokens = annotate(Code: "SomeStruct {\n"
3254	"#if defined FOO\n"
3255	" .a = 1,\n"
3256	"#endif\n"
3257	" .b = 2\n"
3258	"};");
3259	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
3260	EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
3261	EXPECT_TOKEN(Tokens[6], tok::period, TT_DesignatedInitializerPeriod);
3262	EXPECT_TOKEN(Tokens[13], tok::period, TT_DesignatedInitializerPeriod);
3263
3264	Tokens = annotate(Code: "Foo foo[] = {[0]{}};");
3265	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3266	EXPECT_TOKEN(Tokens[6], tok::l_square, TT_DesignatedInitializerLSquare);
3267	EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
3268	}
3269
3270	TEST_F(TokenAnnotatorTest, UnderstandsJavaScript) {
3271	auto Annotate = [this](StringRef Code) {
3272	return annotate(Code, Style: getLLVMStyle(Language: FormatStyle::LK_JavaScript));
3273	};
3274
3275	// Dictionary.
3276	auto Tokens = Annotate("var x = {'x' : 1, 'y' : 2};");
3277	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3278	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_DictLiteral);
3279	EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_SelectorName);
3280	EXPECT_TOKEN(Tokens[5], tok::colon, TT_DictLiteral);
3281	EXPECT_TOKEN(Tokens[8], tok::string_literal, TT_SelectorName);
3282	EXPECT_TOKEN(Tokens[9], tok::colon, TT_DictLiteral);
3283	// Change when we need to annotate these.
3284	EXPECT_BRACE_KIND(Tokens[3], BK_Unknown);
3285	EXPECT_BRACE_KIND(Tokens[11], BK_Unknown);
3286	EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_Unknown);
3287	}
3288
3289	TEST_F(TokenAnnotatorTest, UnderstandsAttributes) {
3290	auto Tokens = annotate(Code: "bool foo __attribute__((unused));");
3291	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3292	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
3293	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_AttributeLParen);
3294	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_Unknown);
3295	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_Unknown);
3296	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_AttributeRParen);
3297
3298	Tokens = annotate(Code: "bool foo __declspec(dllimport);");
3299	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3300	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_AttributeLParen);
3301	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);
3302
3303	Tokens = annotate(Code: "bool __attribute__((unused)) foo;");
3304	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3305	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_AttributeLParen);
3306	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_Unknown);
3307	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_Unknown);
3308	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_AttributeRParen);
3309	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_StartOfName);
3310
3311	Tokens = annotate(Code: "void __attribute__((x)) Foo();");
3312	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3313	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_AttributeLParen);
3314	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_Unknown);
3315	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_Unknown);
3316	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_AttributeRParen);
3317	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_FunctionDeclarationName);
3318	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_FunctionDeclarationLParen);
3319
3320	FormatStyle Style = getLLVMStyle();
3321	Style.AttributeMacros.push_back(x: "FOO");
3322	Tokens = annotate(Code: "bool foo FOO(unused);", Style);
3323	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3324	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_AttributeMacro);
3325	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_AttributeLParen);
3326	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);
3327	}
3328
3329	TEST_F(TokenAnnotatorTest, UnderstandsNullabilityAttributeMacros) {
3330	// Under Google style, handles the Abseil macro aliases for the Clang
3331	// nullability annotations.
3332	auto Style = getGoogleStyle(Language: FormatStyle::LK_Cpp);
3333	auto Tokens = annotate(Code: "x = (foo* absl_nullable)*v;", Style);
3334	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3335	EXPECT_TOKEN(Tokens[4], tok::star, TT_PointerOrReference);
3336	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_AttributeMacro);
3337	EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
3338
3339	Tokens = annotate(Code: "x = (foo* absl_nonnull)*v;", Style);
3340	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3341	EXPECT_TOKEN(Tokens[4], tok::star, TT_PointerOrReference);
3342	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_AttributeMacro);
3343	EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
3344
3345	Tokens = annotate(Code: "x = (foo* absl_nullability_unknown)*v;", Style);
3346	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3347	EXPECT_TOKEN(Tokens[4], tok::star, TT_PointerOrReference);
3348	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_AttributeMacro);
3349	EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
3350	}
3351
3352	TEST_F(TokenAnnotatorTest, UnderstandsControlStatements) {
3353	auto Tokens = annotate(Code: "while (true) {}");
3354	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3355	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);
3356	EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);
3357
3358	Tokens = annotate(Code: "for (;;) {}");
3359	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3360	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_ControlStatementLBrace);
3361	EXPECT_TOKEN(Tokens[6], tok::r_brace, TT_ControlStatementRBrace);
3362
3363	Tokens = annotate(Code: "do {} while (true);");
3364	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3365	EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_ControlStatementLBrace);
3366	EXPECT_TOKEN(Tokens[2], tok::r_brace, TT_ControlStatementRBrace);
3367
3368	Tokens = annotate(Code: "if (true) {} else if (false) {} else {}");
3369	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
3370	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);
3371	EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);
3372	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_ControlStatementLBrace);
3373	EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_ControlStatementRBrace);
3374	EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_ElseLBrace);
3375	EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_ElseRBrace);
3376
3377	Tokens = annotate(Code: "switch (foo) {}");
3378	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3379	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);
3380	EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);
3381	}
3382
3383	TEST_F(TokenAnnotatorTest, UnderstandsDoWhile) {
3384	auto Tokens = annotate(Code: "do { ++i; } while ( i > 5 );");
3385	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3386	EXPECT_TOKEN(Tokens[6], tok::kw_while, TT_DoWhile);
3387
3388	Tokens = annotate(Code: "do ++i; while ( i > 5 );");
3389	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3390	EXPECT_TOKEN(Tokens[4], tok::kw_while, TT_DoWhile);
3391	}
3392
3393	TEST_F(TokenAnnotatorTest, StartOfName) {
3394	auto Tokens = annotate(Code: "#pragma clang diagnostic push");
3395	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3396	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown);
3397	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
3398	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);
3399
3400	Tokens = annotate(Code: "#pragma clang diagnostic ignored \"-Wzero-length-array\"");
3401	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3402	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown);
3403	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
3404	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);
3405
3406	Tokens = annotate(Code: "#define FOO Foo foo");
3407	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3408	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown);
3409	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
3410	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_StartOfName);
3411
3412	Tokens = annotate(Code: "@interface NSCoder (TestCoder)");
3413	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3414	EXPECT_TOKEN(Tokens[0], tok::at, TT_ObjCDecl);
3415	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_StartOfName);
3416
3417	Tokens = annotate(Code: "class FOO BAR C {};");
3418	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3419	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown); // Not StartOfName
3420
3421	auto Style = getLLVMStyle();
3422	Style.StatementAttributeLikeMacros.push_back(x: "emit");
3423	Tokens = annotate(Code: "emit foo = 0;", Style);
3424	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3425	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_StatementAttributeLikeMacro);
3426	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);
3427	}
3428
3429	TEST_F(TokenAnnotatorTest, BraceKind) {
3430	auto Tokens = annotate(Code: "void f() {};");
3431	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3432	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3433	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3434	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
3435	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3436	EXPECT_BRACE_KIND(Tokens[5], BK_Block);
3437
3438	Tokens = annotate(Code: "class Foo<int> f() {}");
3439	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3440	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);
3441	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_FunctionDeclarationName);
3442	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionDeclarationLParen);
3443	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);
3444	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3445	EXPECT_BRACE_KIND(Tokens[9], BK_Block);
3446
3447	Tokens = annotate(Code: "template <typename T> class Foo<T> f() {}");
3448	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
3449	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
3450	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_FunctionDeclarationName);
3451	EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);
3452	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
3453	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3454	EXPECT_BRACE_KIND(Tokens[14], BK_Block);
3455
3456	Tokens = annotate(Code: "void f() override {};");
3457	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3458	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3459	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3460	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);
3461	EXPECT_BRACE_KIND(Tokens[5], BK_Block);
3462	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3463
3464	Tokens = annotate(Code: "void f() noexcept(false) {};");
3465	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3466	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3467	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3468	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);
3469	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3470	EXPECT_BRACE_KIND(Tokens[9], BK_Block);
3471
3472	Tokens = annotate(Code: "auto f() -> void {};");
3473	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3474	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3475	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3476	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);
3477	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3478	EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3479
3480	Tokens = annotate(Code: "void f() { /**/ };");
3481	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3482	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3483	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3484	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
3485	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3486	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3487
3488	Tokens = annotate(Code: "void f() { //\n"
3489	"};");
3490	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3491	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3492	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3493	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
3494	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3495	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3496
3497	Tokens = annotate(Code: "void f() {\n"
3498	" //\n"
3499	"};");
3500	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3501	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
3502	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
3503	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
3504	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3505	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3506
3507	Tokens = annotate(Code: "struct Foo {\n"
3508	" Foo() {};\n"
3509	" ~Foo() {};\n"
3510	"};");
3511	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
3512	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
3513	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);
3514	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);
3515	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3516	EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3517	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_CtorDtorDeclName);
3518	EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);
3519	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
3520	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3521	EXPECT_BRACE_KIND(Tokens[14], BK_Block);
3522
3523	Tokens = annotate(Code: "{\n"
3524	" char *a[] = {\n"
3525	" /* abc */ \"abc\",\n"
3526	"#if FOO\n"
3527	" /* xyz */ \"xyz\",\n"
3528	"#endif\n"
3529	" /* last */ \"last\"};\n"
3530	"}");
3531	ASSERT_EQ(Tokens.size(), 25u) << Tokens;
3532	EXPECT_BRACE_KIND(Tokens[0], BK_Block);
3533	EXPECT_BRACE_KIND(Tokens[7], BK_BracedInit);
3534	EXPECT_BRACE_KIND(Tokens[21], BK_BracedInit);
3535
3536	Tokens =
3537	annotate(Code: "#define SCOP_STAT(NAME, DESC) \\\n"
3538	" {\"polly\", #NAME, \"Number of rejected regions: \" DESC}");
3539	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
3540	EXPECT_BRACE_KIND(Tokens[8], BK_BracedInit);
3541	EXPECT_BRACE_KIND(Tokens[16], BK_BracedInit);
3542
3543	Tokens = annotate(Code: "struct {};");
3544	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
3545	EXPECT_BRACE_KIND(Tokens[1], BK_Block);
3546	EXPECT_BRACE_KIND(Tokens[2], BK_Block);
3547
3548	Tokens = annotate(Code: "struct : Base {};");
3549	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3550	EXPECT_BRACE_KIND(Tokens[3], BK_Block);
3551	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3552
3553	Tokens = annotate(Code: "struct Foo {};");
3554	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3555	EXPECT_BRACE_KIND(Tokens[2], BK_Block);
3556	EXPECT_BRACE_KIND(Tokens[3], BK_Block);
3557
3558	Tokens = annotate(Code: "struct ::Foo {};");
3559	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3560	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);
3561	EXPECT_BRACE_KIND(Tokens[3], BK_Block);
3562	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3563
3564	Tokens = annotate(Code: "struct NS::Foo {};");
3565	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3566	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);
3567	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_ClassHeadName);
3568	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3569	EXPECT_BRACE_KIND(Tokens[5], BK_Block);
3570
3571	Tokens = annotate(Code: "struct Foo<int> {};");
3572	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3573	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
3574	EXPECT_BRACE_KIND(Tokens[5], BK_Block);
3575	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
3576
3577	Tokens = annotate(Code: "struct Foo<int>::Bar {};");
3578	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3579	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown);
3580	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_ClassHeadName);
3581	EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3582	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3583
3584	Tokens = annotate(Code: "struct Foo<int> : Base {};");
3585	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3586	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
3587	EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3588	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3589
3590	Tokens = annotate(Code: "struct Foo<int> : Base::Bar {};");
3591	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
3592	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
3593	EXPECT_TOKEN(Tokens[8], tok::identifier, TT_Unknown); // Not TT_ClassHeadName.
3594	EXPECT_BRACE_KIND(Tokens[9], BK_Block);
3595	EXPECT_BRACE_KIND(Tokens[10], BK_Block);
3596
3597	Tokens = annotate(Code: "struct Foo final {};");
3598	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3599	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_ClassHeadName);
3600	EXPECT_BRACE_KIND(Tokens[3], BK_Block);
3601	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3602
3603	Tokens = annotate(Code: "struct [[foo]] [[bar]] Foo final : Base1, Base2 {};");
3604	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
3605	EXPECT_TOKEN(Tokens[11], tok::identifier, TT_ClassHeadName);
3606	EXPECT_BRACE_KIND(Tokens[17], BK_Block);
3607	EXPECT_BRACE_KIND(Tokens[18], BK_Block);
3608
3609	Tokens = annotate(Code: "struct Foo x{};");
3610	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3611	EXPECT_BRACE_KIND(Tokens[3], BK_BracedInit);
3612	EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);
3613
3614	Tokens = annotate(Code: "struct ::Foo x{};");
3615	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3616	EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);
3617	EXPECT_BRACE_KIND(Tokens[5], BK_BracedInit);
3618
3619	Tokens = annotate(Code: "struct NS::Foo x{};");
3620	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3621	EXPECT_BRACE_KIND(Tokens[5], BK_BracedInit);
3622	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
3623
3624	Tokens = annotate(Code: "struct Foo<int> x{};");
3625	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3626	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
3627	EXPECT_BRACE_KIND(Tokens[7], BK_BracedInit);
3628
3629	Tokens = annotate(Code: "#ifdef DEBUG_ENABLED\n"
3630	"#else\n"
3631	"#endif\n"
3632	"class RenderingServer : Object {\n"
3633	"#ifndef DISABLE_DEPRECATED\n"
3634	" enum Features {\n"
3635	" FEATURE_SHADERS,\n"
3636	" FEATURE_MULTITHREADED,\n"
3637	" };\n"
3638	"#endif\n"
3639	"};");
3640	ASSERT_EQ(Tokens.size(), 29u) << Tokens;
3641	EXPECT_TOKEN(Tokens[8], tok::identifier, TT_ClassHeadName);
3642	EXPECT_BRACE_KIND(Tokens[11], BK_Block);
3643	EXPECT_BRACE_KIND(Tokens[17], BK_Block);
3644	EXPECT_BRACE_KIND(Tokens[22], BK_Block);
3645	EXPECT_BRACE_KIND(Tokens[26], BK_Block);
3646
3647	Tokens = annotate(Code: "{\n"
3648	"#define M(x) \\\n"
3649	" return {#x};\n"
3650	"}");
3651	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
3652	EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_BlockLBrace);
3653	EXPECT_BRACE_KIND(Tokens[0], BK_Block);
3654	EXPECT_BRACE_KIND(Tokens[8], BK_BracedInit);
3655	EXPECT_BRACE_KIND(Tokens[11], BK_BracedInit);
3656	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3657
3658	Tokens = annotate(Code: "{\n"
3659	" {\n"
3660	"#define GEN_ID(_x) char *_x{#_x}\n"
3661	" GEN_ID(one);\n"
3662	" }\n"
3663	"}");
3664	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
3665	EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_BlockLBrace);
3666	EXPECT_BRACE_KIND(Tokens[0], BK_Block);
3667	EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_BlockLBrace);
3668	EXPECT_BRACE_KIND(Tokens[1], BK_Block);
3669	EXPECT_BRACE_KIND(Tokens[11], BK_BracedInit);
3670	EXPECT_BRACE_KIND(Tokens[14], BK_BracedInit);
3671	EXPECT_BRACE_KIND(Tokens[20], BK_Block);
3672	EXPECT_BRACE_KIND(Tokens[21], BK_Block);
3673
3674	Tokens = annotate(Code: "{\n"
3675	"#define FOO \\\n"
3676	" { \\\n"
3677	" case bar: { \\\n"
3678	" break; \\\n"
3679	" } \\\n"
3680	" }\n"
3681	"}");
3682	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
3683	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_BlockLBrace);
3684	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3685	EXPECT_TOKEN(Tokens[7], tok::colon, TT_CaseLabelColon);
3686	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3687	EXPECT_BRACE_KIND(Tokens[11], BK_Block);
3688	EXPECT_BRACE_KIND(Tokens[12], BK_Block);
3689
3690	Tokens = annotate(Code: "class foo {\n"
3691	" foo() {}\n"
3692	"#if defined(_MSC_VER__clang____GNUC__FOO_) || \\\n"
3693	" (defined(__GNUC__) && defined(FOO))\n"
3694	" foo() {}\n"
3695	"#endif\n"
3696	"};");
3697	ASSERT_EQ(Tokens.size(), 36u) << Tokens;
3698	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);
3699	EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3700	EXPECT_TOKEN(Tokens[26], tok::identifier, TT_CtorDtorDeclName);
3701	EXPECT_TOKEN(Tokens[27], tok::l_paren, TT_FunctionDeclarationLParen);
3702
3703	Tokens = annotate(Code: "a = class extends goog.a {};",
3704	Style: getGoogleStyle(Language: FormatStyle::LK_JavaScript));
3705	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3706	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
3707	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_ClassLBrace);
3708	EXPECT_BRACE_KIND(Tokens[7], BK_Block);
3709	EXPECT_TOKEN(Tokens[8], tok::r_brace, TT_ClassRBrace);
3710	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3711
3712	Tokens = annotate(Code: "a = class Foo extends goog.a {};",
3713	Style: getGoogleStyle(Language: FormatStyle::LK_JavaScript));
3714	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3715	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_ClassHeadName);
3716	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown); // Not TT_StartOfName
3717	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_ClassLBrace);
3718	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3719	EXPECT_TOKEN(Tokens[9], tok::r_brace, TT_ClassRBrace);
3720	EXPECT_BRACE_KIND(Tokens[9], BK_Block);
3721
3722	Tokens = annotate(Code: "#define FOO(X) \\\n"
3723	" struct X##_tag_ {};");
3724	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3725	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_Unknown);
3726	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_ClassHeadName);
3727	EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_StructLBrace);
3728	EXPECT_BRACE_KIND(Tokens[10], BK_Block);
3729	EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_StructRBrace);
3730	EXPECT_BRACE_KIND(Tokens[11], BK_Block);
3731
3732	Tokens = annotate(Code: "#define MACRO \\\n"
3733	" struct hash<type> { \\\n"
3734	" void f() { return; } \\\n"
3735	" };");
3736	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
3737	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_ClassHeadName);
3738	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_StructLBrace);
3739	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
3740	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_FunctionDeclarationName);
3741	EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_FunctionDeclarationLParen);
3742	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
3743	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
3744	EXPECT_BRACE_KIND(Tokens[16], BK_Block);
3745	EXPECT_TOKEN(Tokens[17], tok::r_brace, TT_StructRBrace);
3746	EXPECT_BRACE_KIND(Tokens[17], BK_Block);
3747
3748	Tokens = annotate(Code: "#define MEMBER(NAME) NAME{\"\"}");
3749	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3750	EXPECT_BRACE_KIND(Tokens[7], BK_BracedInit);
3751	EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
3752
3753	Tokens = annotate(Code: "return lhs ^ Byte{rhs};");
3754	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3755	EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);
3756	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
3757	}
3758
3759	TEST_F(TokenAnnotatorTest, UnderstandsElaboratedTypeSpecifier) {
3760	auto Tokens = annotate(Code: "auto foo() -> enum En {}");
3761	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
3762	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
3763	}
3764
3765	TEST_F(TokenAnnotatorTest, BlockLBrace) {
3766	auto Tokens = annotate(Code: "{\n"
3767	" {\n"
3768	" foo();\n"
3769	" }\n"
3770	"}");
3771	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
3772	EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_BlockLBrace);
3773	EXPECT_BRACE_KIND(Tokens[0], BK_Block);
3774	EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_BlockLBrace);
3775	EXPECT_BRACE_KIND(Tokens[1], BK_Block);
3776
3777	Tokens = annotate(Code: "void bar() {\n"
3778	" {\n"
3779	" foo();\n"
3780	" }\n"
3781	"}");
3782	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
3783	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
3784	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
3785	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_BlockLBrace);
3786	EXPECT_BRACE_KIND(Tokens[5], BK_Block);
3787
3788	Tokens = annotate(Code: "[foo bar:{{0, 1}} baz:baz];",
3789	Style: getLLVMStyle(Language: FormatStyle::LK_ObjC));
3790	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
3791	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_Unknown); // Not TT_BlockLBrace.
3792	EXPECT_BRACE_KIND(Tokens[4], BK_Unknown); // Not BK_Block.
3793	EXPECT_BRACE_KIND(Tokens[5], BK_BracedInit);
3794	EXPECT_BRACE_KIND(Tokens[9], BK_Unknown); // Not BK_Block.
3795	EXPECT_BRACE_KIND(Tokens[10], BK_Unknown); // Not BK_Block.
3796	}
3797
3798	TEST_F(TokenAnnotatorTest, SwitchExpression) {
3799	auto Style = getLLVMStyle(Language: FormatStyle::LK_Java);
3800	auto Tokens = annotate(Code: "i = switch (day) {\n"
3801	" case THURSDAY, SATURDAY -> 8;\n"
3802	" case WEDNESDAY -> 9;\n"
3803	" default -> 1;\n"
3804	"};",
3805	Style);
3806	ASSERT_EQ(Tokens.size(), 26u) << Tokens;
3807	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_SwitchExpressionLBrace);
3808	EXPECT_TOKEN(Tokens[7], tok::kw_case, TT_SwitchExpressionLabel);
3809	EXPECT_TOKEN(Tokens[11], tok::arrow, TT_CaseLabelArrow);
3810	EXPECT_TOKEN(Tokens[14], tok::kw_case, TT_SwitchExpressionLabel);
3811	EXPECT_TOKEN(Tokens[16], tok::arrow, TT_CaseLabelArrow);
3812	EXPECT_TOKEN(Tokens[19], tok::kw_default, TT_SwitchExpressionLabel);
3813	EXPECT_TOKEN(Tokens[20], tok::arrow, TT_CaseLabelArrow);
3814	}
3815
3816	TEST_F(TokenAnnotatorTest, JavaRecord) {
3817	auto Tokens = annotate(Code: "public record MyRecord() {}",
3818	Style: getLLVMStyle(Language: FormatStyle::LK_Java));
3819	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3820	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_ClassHeadName);
3821	// Not TT_FunctionDeclarationLParen.
3822	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_Unknown);
3823	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_RecordLBrace);
3824	EXPECT_TOKEN(Tokens[6], tok::r_brace, TT_RecordRBrace);
3825	}
3826
3827	TEST_F(TokenAnnotatorTest, CppAltOperatorKeywords) {
3828	auto Tokens = annotate(Code: "a = b and c;");
3829	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3830	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator);
3831
3832	Tokens = annotate(Code: "a = b and_eq c;");
3833	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3834	EXPECT_TOKEN(Tokens[3], tok::ampequal, TT_BinaryOperator);
3835
3836	Tokens = annotate(Code: "a = b bitand c;");
3837	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3838	EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
3839
3840	Tokens = annotate(Code: "a = b bitor c;");
3841	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3842	EXPECT_TOKEN(Tokens[3], tok::pipe, TT_BinaryOperator);
3843
3844	Tokens = annotate(Code: "a = b compl c;");
3845	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3846	EXPECT_TOKEN(Tokens[3], tok::tilde, TT_UnaryOperator);
3847
3848	Tokens = annotate(Code: "a = b not c;");
3849	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3850	EXPECT_TOKEN(Tokens[3], tok::exclaim, TT_UnaryOperator);
3851
3852	Tokens = annotate(Code: "a = b not_eq c;");
3853	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3854	EXPECT_TOKEN(Tokens[3], tok::exclaimequal, TT_BinaryOperator);
3855
3856	Tokens = annotate(Code: "a = b or c;");
3857	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3858	EXPECT_TOKEN(Tokens[3], tok::pipepipe, TT_BinaryOperator);
3859
3860	Tokens = annotate(Code: "return segment < *this or *this < segment;");
3861	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3862	EXPECT_TOKEN(Tokens[5], tok::pipepipe, TT_BinaryOperator);
3863	EXPECT_TOKEN(Tokens[6], tok::star, TT_UnaryOperator);
3864
3865	Tokens = annotate(Code: "a = b or_eq c;");
3866	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3867	EXPECT_TOKEN(Tokens[3], tok::pipeequal, TT_BinaryOperator);
3868
3869	Tokens = annotate(Code: "a = b xor c;");
3870	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3871	EXPECT_TOKEN(Tokens[3], tok::caret, TT_BinaryOperator);
3872
3873	Tokens = annotate(Code: "a = b xor_eq c;");
3874	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
3875	EXPECT_TOKEN(Tokens[3], tok::caretequal, TT_BinaryOperator);
3876
3877	const auto StyleC = getLLVMStyle(Language: FormatStyle::LK_C);
3878
3879	Tokens = annotate(Code: "xor = foo;", Style: StyleC);
3880	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
3881	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
3882
3883	Tokens = annotate(Code: "int xor = foo;", Style: StyleC);
3884	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3885	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
3886	}
3887
3888	TEST_F(TokenAnnotatorTest, CppOnlyKeywordInC) {
3889	auto Tokens = annotate(Code: "int maximized = new & STATE_MAXIMIZED;",
3890	Style: getLLVMStyle(Language: FormatStyle::LK_C));
3891	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
3892	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown); // Not tok::kw_new
3893	EXPECT_TOKEN(Tokens[4], tok::amp, TT_BinaryOperator);
3894	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown); // Not TT_StartOfName
3895	}
3896
3897	TEST_F(TokenAnnotatorTest, FunctionTryBlock) {
3898	auto Tokens =
3899	annotate(Code: "Foo::Foo(int x, int y) try\n"
3900	" : foo{[] -> std::string { return {}; }(), x}, bar{y} {\n"
3901	"} catch (...) {\n"
3902	"}");
3903	ASSERT_EQ(Tokens.size(), 45u) << Tokens;
3904	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_CtorDtorDeclName);
3905	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);
3906	EXPECT_TOKEN(Tokens[11], tok::colon, TT_CtorInitializerColon);
3907	EXPECT_TOKEN(Tokens[14], tok::l_square, TT_LambdaLSquare);
3908	EXPECT_TOKEN(Tokens[16], tok::arrow, TT_LambdaArrow);
3909	EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
3910	EXPECT_TOKEN(Tokens[31], tok::comma, TT_CtorInitializerComma);
3911	EXPECT_TOKEN(Tokens[36], tok::l_brace, TT_FunctionLBrace);
3912	}
3913
3914	TEST_F(TokenAnnotatorTest, TypenameMacro) {
3915	auto Style = getLLVMStyle();
3916	Style.TypenameMacros.push_back(x: "STRUCT");
3917
3918	auto Tokens = annotate(Code: "STRUCT(T, B) { int i; };", Style);
3919	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
3920	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TypenameMacro);
3921	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_TypeDeclarationParen);
3922	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_TypeDeclarationParen);
3923	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_Unknown);
3924	}
3925
3926	TEST_F(TokenAnnotatorTest, GNULanguageStandard) {
3927	auto Style = getGNUStyle();
3928	EXPECT_EQ(Style.Standard, FormatStyle::LS_Latest);
3929
3930	auto Tokens = annotate(Code: "return 1 <=> 2;", Style);
3931	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
3932	EXPECT_TOKEN(Tokens[2], tok::spaceship, TT_BinaryOperator);
3933	}
3934
3935	TEST_F(TokenAnnotatorTest, SplitPenalty) {
3936	auto Style = getLLVMStyle();
3937	Style.ColumnLimit = 20;
3938
3939	auto Tokens = annotate(Code: "class foo {\n"
3940	" auto bar()\n"
3941	" -> bool;\n"
3942	"};",
3943	Style);
3944	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
3945	EXPECT_TOKEN(Tokens[7], tok::arrow, TT_TrailingReturnArrow);
3946	EXPECT_SPLIT_PENALTY(Tokens[7], 23u);
3947	}
3948
3949	TEST_F(TokenAnnotatorTest, TemplateName) {
3950	constexpr StringRef Code{"return Foo < A || B > (C ^ D);"};
3951
3952	auto Tokens = annotate(Code);
3953	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
3954	EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
3955	EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);
3956
3957	auto Style = getLLVMStyle();
3958	Style.TemplateNames.push_back(x: "Foo");
3959
3960	Tokens = annotate(Code, Style);
3961	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_TemplateName);
3962	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
3963	EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
3964	}
3965
3966	TEST_F(TokenAnnotatorTest, TemplateInstantiation) {
3967	auto Tokens = annotate(Code: "return FixedInt<N | M>();");
3968	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
3969	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
3970	EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
3971
3972	Tokens = annotate(Code: "return FixedInt<N | M>(foo);");
3973	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
3974	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
3975	EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
3976
3977	Tokens = annotate(Code: "return std::conditional_t<T::value == U::value, T, U>{};");
3978	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
3979	EXPECT_TOKEN(Tokens[4], tok::less, TT_TemplateOpener);
3980	EXPECT_TOKEN(Tokens[16], tok::greater, TT_TemplateCloser);
3981
3982	Tokens =
3983	annotate(Code: "auto x{std::conditional_t<T::value == U::value, T, U>{}};");
3984	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
3985	EXPECT_TOKEN(Tokens[6], tok::less, TT_TemplateOpener);
3986	EXPECT_TOKEN(Tokens[18], tok::greater, TT_TemplateCloser);
3987
3988	Tokens = annotate(
3989	Code: "std::uint16_t kMTU = std::conditional<\n"
3990	" kTypeKind == KindA,\n"
3991	" std::integral_constant<std::uint16_t, kIoSockMtu>>::type::value;");
3992	ASSERT_EQ(Tokens.size(), 30u) << Tokens;
3993	EXPECT_TOKEN(Tokens[8], tok::less, TT_TemplateOpener);
3994	EXPECT_TOKEN(Tokens[16], tok::less, TT_TemplateOpener);
3995	EXPECT_TOKEN(Tokens[22], tok::greater, TT_TemplateCloser);
3996	EXPECT_TOKEN(Tokens[23], tok::greater, TT_TemplateCloser);
3997	}
3998
3999	TEST_F(TokenAnnotatorTest, VariableTemplate) {
4000	auto Style = getLLVMStyle();
4001	Style.VariableTemplates.push_back(x: "a");
4002
4003	auto Tokens = annotate(Code: "auto t3 = (a<int>) + b;", Style);
4004	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
4005	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_VariableTemplate);
4006	EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);
4007	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
4008	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown); // Not TT_CastRParen
4009	EXPECT_TOKEN(Tokens[9], tok::plus, TT_BinaryOperator);
4010	}
4011
4012	TEST_F(TokenAnnotatorTest, SwitchInMacroArgument) {
4013	auto Tokens = annotate(Code: "FOOBAR(switch);\n"
4014	"void f() {}");
4015	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
4016	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_FunctionLBrace);
4017	}
4018
4019	TEST_F(TokenAnnotatorTest, AfterPPDirective) {
4020	auto Tokens = annotate(Code: "#error -- My error message");
4021
4022	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
4023	EXPECT_TOKEN(Tokens[2], tok::minusminus, TT_AfterPPDirective);
4024	}
4025
4026	TEST_F(TokenAnnotatorTest, UserDefinedConversionFunction) {
4027	auto Tokens = annotate(Code: "operator int(void);");
4028	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
4029	EXPECT_TOKEN(Tokens[0], tok::kw_operator, TT_FunctionDeclarationName);
4030	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
4031
4032	Tokens = annotate(Code: "explicit operator int *();");
4033	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
4034	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
4035	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
4036	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionDeclarationLParen);
4037
4038	Tokens = annotate(Code: "operator int &();");
4039	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
4040	EXPECT_TOKEN(Tokens[0], tok::kw_operator, TT_FunctionDeclarationName);
4041	EXPECT_TOKEN(Tokens[2], tok::amp, TT_PointerOrReference);
4042	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);
4043
4044	Tokens = annotate(Code: "operator auto() const { return 2; }");
4045	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
4046	EXPECT_TOKEN(Tokens[0], tok::kw_operator, TT_FunctionDeclarationName);
4047	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
4048	EXPECT_TOKEN(Tokens[4], tok::kw_const, TT_TrailingAnnotation);
4049	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);
4050
4051	Tokens = annotate(Code: "operator decltype(auto)() const;");
4052	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
4053	EXPECT_TOKEN(Tokens[0], tok::kw_operator, TT_FunctionDeclarationName);
4054	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_TypeDeclarationParen);
4055	EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_TypeDeclarationParen);
4056	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_FunctionDeclarationLParen);
4057	EXPECT_TOKEN(Tokens[7], tok::kw_const, TT_TrailingAnnotation);
4058
4059	Tokens = annotate(Code: "virtual operator Foo() = 0;");
4060	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
4061	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
4062	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_FunctionDeclarationLParen);
4063
4064	Tokens = annotate(Code: "operator Foo() override { return Foo(); }");
4065	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
4066	EXPECT_TOKEN(Tokens[0], tok::kw_operator, TT_FunctionDeclarationName);
4067	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_FunctionDeclarationLParen);
4068	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);
4069
4070	Tokens = annotate(Code: "friend Bar::operator Foo();");
4071	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
4072	EXPECT_TOKEN(Tokens[3], tok::kw_operator, TT_FunctionDeclarationName);
4073	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_FunctionDeclarationLParen);
4074	}
4075
4076	TEST_F(TokenAnnotatorTest, UTF8StringLiteral) {
4077	auto Tokens = annotate(Code: "return u8\"foo\";", Style: getLLVMStyle(Language: FormatStyle::LK_C));
4078	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
4079	EXPECT_TOKEN(Tokens[1], tok::utf8_string_literal, TT_Unknown);
4080	}
4081
4082	TEST_F(TokenAnnotatorTest, IdentifierPackage) {
4083	auto Tokens = annotate(Code: "auto package;");
4084	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
4085	EXPECT_FALSE(Tokens[0]->isObjCAccessSpecifier());
4086	}
4087
4088	TEST_F(TokenAnnotatorTest, UserDefinedLiteral) {
4089	auto Tokens = annotate(Code: "auto dollars = 2_$;");
4090	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
4091	EXPECT_EQ(Tokens[3]->TokenText, "2_$");
4092	}
4093
4094	TEST_F(TokenAnnotatorTest, EnumColonInTypedef) {
4095	auto Tokens = annotate(Code: "typedef enum : int {} foo;");
4096	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
4097	EXPECT_TOKEN(Tokens[2], tok::colon, TT_Unknown); // Not TT_InheritanceColon.
4098	}
4099
4100	TEST_F(TokenAnnotatorTest, BitFieldColon) {
4101	auto Tokens = annotate(Code: "class C {\n"
4102	" int f : SIZE;\n"
4103	"};");
4104	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
4105	EXPECT_TOKEN(Tokens[5], tok::colon, TT_BitFieldColon);
4106	}
4107
4108	TEST_F(TokenAnnotatorTest, JsonCodeInRawString) {
4109	auto Tokens = annotate(Code: "{\n"
4110	" \"foo\": \"bar\",\n"
4111	" \"str\": \"test\"\n"
4112	"}",
4113	Style: getLLVMStyle(Language: FormatStyle::LK_Json));
4114	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
4115	EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_DictLiteral);
4116	EXPECT_TOKEN(Tokens[1], tok::string_literal, TT_SelectorName);
4117	EXPECT_TOKEN(Tokens[2], tok::colon, TT_DictLiteral);
4118	EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_SelectorName);
4119	EXPECT_TOKEN(Tokens[6], tok::colon, TT_DictLiteral);
4120	}
4121
4122	} // namespace
4123	} // namespace format
4124	} // namespace clang
4125