1//===-- DILLexerTests.cpp --------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "lldb/ValueObject/DILLexer.h"
10#include "llvm/ADT/StringRef.h"
11#include "llvm/Testing/Support/Error.h"
12#include "gtest/gtest.h"
13#include <string>
14
15using llvm::StringRef;
16
17using namespace lldb_private::dil;
18
19llvm::Expected<std::vector<std::pair<Token::Kind, std::string>>>
20ExtractTokenData(llvm::StringRef input_expr) {
21
22 llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(expr: input_expr);
23 if (!maybe_lexer)
24 return maybe_lexer.takeError();
25 DILLexer lexer(*maybe_lexer);
26
27 std::vector<std::pair<Token::Kind, std::string>> data;
28 do {
29 Token tok = lexer.GetCurrentToken();
30 data.push_back(x: std::make_pair(x: tok.GetKind(), y: tok.GetSpelling()));
31 lexer.Advance();
32 } while (data.back().first != Token::eof);
33 // Don't return the eof token.
34 data.pop_back();
35 return data;
36}
37
38TEST(DILLexerTests, SimpleTest) {
39 StringRef input_expr("simple_var");
40 llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(expr: input_expr);
41 ASSERT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
42 DILLexer lexer(*maybe_lexer);
43 Token token = lexer.GetCurrentToken();
44
45 EXPECT_EQ(token.GetKind(), Token::identifier);
46 EXPECT_EQ(token.GetSpelling(), "simple_var");
47 lexer.Advance();
48 token = lexer.GetCurrentToken();
49 EXPECT_EQ(token.GetKind(), Token::eof);
50}
51
52TEST(DILLexerTests, TokenKindTest) {
53 Token token = Token(Token::identifier, "ident", 0);
54
55 EXPECT_TRUE(token.Is(Token::identifier));
56 EXPECT_FALSE(token.Is(Token::l_paren));
57 EXPECT_TRUE(token.IsOneOf({Token::eof, Token::identifier}));
58 EXPECT_FALSE(token.IsOneOf(
59 {Token::l_paren, Token::r_paren, Token::coloncolon, Token::eof}));
60}
61
62TEST(DILLexerTests, LookAheadTest) {
63 StringRef input_expr("(anonymous namespace)::some_var");
64 llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(expr: input_expr);
65 ASSERT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
66 DILLexer lexer(*maybe_lexer);
67 Token token = lexer.GetCurrentToken();
68
69 // Current token is '('; check the next 4 tokens, to make
70 // sure they are the identifier 'anonymous', the identifier 'namespace'
71 // ')' and '::', in that order.
72 EXPECT_EQ(token.GetKind(), Token::l_paren);
73 EXPECT_EQ(lexer.LookAhead(1).GetKind(), Token::identifier);
74 EXPECT_EQ(lexer.LookAhead(1).GetSpelling(), "anonymous");
75 EXPECT_EQ(lexer.LookAhead(2).GetKind(), Token::identifier);
76 EXPECT_EQ(lexer.LookAhead(2).GetSpelling(), "namespace");
77 EXPECT_EQ(lexer.LookAhead(3).GetKind(), Token::r_paren);
78 EXPECT_EQ(lexer.LookAhead(4).GetKind(), Token::coloncolon);
79
80 // Our current index should still be 0, as we only looked ahead; we are still
81 // officially on the '('.
82 EXPECT_EQ(lexer.GetCurrentTokenIdx(), 0u);
83
84 // Accept the 'lookahead', so our current token is '::', which has the index
85 // 4 in our vector of tokens (which starts at zero).
86 lexer.Advance(N: 4);
87 token = lexer.GetCurrentToken();
88 EXPECT_EQ(token.GetKind(), Token::coloncolon);
89 EXPECT_EQ(lexer.GetCurrentTokenIdx(), 4u);
90
91 lexer.Advance();
92 token = lexer.GetCurrentToken();
93 EXPECT_EQ(token.GetKind(), Token::identifier);
94 EXPECT_EQ(token.GetSpelling(), "some_var");
95 EXPECT_EQ(lexer.GetCurrentTokenIdx(), 5u);
96 EXPECT_EQ(token.GetLocation(), strlen("(anonymous namespace)::"));
97
98 lexer.Advance();
99 token = lexer.GetCurrentToken();
100 EXPECT_EQ(token.GetKind(), Token::eof);
101}
102
103TEST(DILLexerTests, MultiTokenLexTest) {
104 EXPECT_THAT_EXPECTED(
105 ExtractTokenData("This string has (several ) ::identifiers"),
106 llvm::HasValue(testing::ElementsAre(
107 testing::Pair(Token::identifier, "This"),
108 testing::Pair(Token::identifier, "string"),
109 testing::Pair(Token::identifier, "has"),
110 testing::Pair(Token::l_paren, "("),
111 testing::Pair(Token::identifier, "several"),
112 testing::Pair(Token::r_paren, ")"),
113 testing::Pair(Token::coloncolon, "::"),
114 testing::Pair(Token::identifier, "identifiers"))));
115}
116
117TEST(DILLexerTests, IdentifiersTest) {
118 // These strings should lex into identifier tokens.
119 std::vector<std::string> valid_identifiers = {
120 "$My_name1", "$pc", "abcd", "_", "_a", "_a_", "$",
121 "a_b", "this", "self", "a", "MyName", "namespace"};
122
123 // The lexer can lex these strings, but they should not be identifiers.
124 std::vector<std::string> invalid_identifiers = {"", "::", "(", ")", "0abc"};
125
126 // The lexer is expected to fail attempting to lex these strings (it cannot
127 // create valid tokens out of them).
128 std::vector<std::string> invalid_tok_strings = {"#include", "a@a"};
129
130 // Verify that all of the valid identifiers come out as identifier tokens.
131 for (auto &str : valid_identifiers) {
132 SCOPED_TRACE(str);
133 EXPECT_THAT_EXPECTED(ExtractTokenData(str),
134 llvm::HasValue(testing::ElementsAre(
135 testing::Pair(Token::identifier, str))));
136 }
137
138 // Verify that the lexer fails on invalid token strings.
139 for (auto &str : invalid_tok_strings) {
140 SCOPED_TRACE(str);
141 auto maybe_lexer = DILLexer::Create(expr: str);
142 EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Failed());
143 }
144
145 // Verify that none of the invalid identifiers come out as identifier tokens.
146 for (auto &str : invalid_identifiers) {
147 SCOPED_TRACE(str);
148 llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(expr: str);
149 EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
150 DILLexer lexer(*maybe_lexer);
151 Token token = lexer.GetCurrentToken();
152 EXPECT_TRUE(token.IsNot(Token::identifier));
153 EXPECT_TRUE(token.IsOneOf({Token::eof, Token::coloncolon, Token::l_paren,
154 Token::r_paren, Token::numeric_constant}));
155 }
156}
157
158TEST(DILLexerTests, NumbersTest) {
159 // These strings should lex into number tokens.
160 std::vector<std::string> valid_numbers = {"123", "0x123", "0123", "0b101"};
161
162 // The lexer can lex these strings, but they should not be numbers.
163 std::vector<std::string> invalid_numbers = {"", "x123", "b123"};
164
165 for (auto &str : valid_numbers) {
166 SCOPED_TRACE(str);
167 EXPECT_THAT_EXPECTED(ExtractTokenData(str),
168 llvm::HasValue(testing::ElementsAre(
169 testing::Pair(Token::numeric_constant, str))));
170 }
171 // Verify that none of the invalid numbers come out as numeric tokens.
172 for (auto &str : invalid_numbers) {
173 SCOPED_TRACE(str);
174 llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(expr: str);
175 EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
176 DILLexer lexer(*maybe_lexer);
177 Token token = lexer.GetCurrentToken();
178 EXPECT_TRUE(token.IsNot(Token::numeric_constant));
179 EXPECT_TRUE(token.IsOneOf({Token::eof, Token::identifier}));
180 }
181}
182

source code of lldb/unittests/ValueObject/DILLexerTests.cpp