1//===- unittests/Basic/SarifTest.cpp - Test writing SARIF documents -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/Basic/Sarif.h"
10#include "clang/Basic/DiagnosticOptions.h"
11#include "clang/Basic/FileManager.h"
12#include "clang/Basic/FileSystemOptions.h"
13#include "clang/Basic/LangOptions.h"
14#include "clang/Basic/SourceLocation.h"
15#include "clang/Basic/SourceManager.h"
16#include "llvm/ADT/StringRef.h"
17#include "llvm/Support/FormatVariadic.h"
18#include "llvm/Support/JSON.h"
19#include "llvm/Support/MemoryBuffer.h"
20#include "llvm/Support/VirtualFileSystem.h"
21#include "llvm/Support/raw_ostream.h"
22#include "gmock/gmock.h"
23#include "gtest/gtest.h"
24
25#include <algorithm>
26
27using namespace clang;
28
29namespace {
30
31using LineCol = std::pair<unsigned int, unsigned int>;
32
33static std::string serializeSarifDocument(llvm::json::Object &&Doc) {
34 std::string Output;
35 llvm::json::Value Value(std::move(Doc));
36 llvm::raw_string_ostream OS{Output};
37 OS << llvm::formatv(Fmt: "{0}", Vals&: Value);
38 OS.flush();
39 return Output;
40}
41
42class SarifDocumentWriterTest : public ::testing::Test {
43protected:
44 SarifDocumentWriterTest()
45 : InMemoryFileSystem(new llvm::vfs::InMemoryFileSystem),
46 FileMgr(FileSystemOptions(), InMemoryFileSystem),
47 DiagID(new DiagnosticIDs()), DiagOpts(new DiagnosticOptions()),
48 Diags(DiagID, DiagOpts.get(), new IgnoringDiagConsumer()),
49 SourceMgr(Diags, FileMgr) {}
50
51 IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem;
52 FileManager FileMgr;
53 IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
54 IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
55 DiagnosticsEngine Diags;
56 SourceManager SourceMgr;
57 LangOptions LangOpts;
58
59 FileID registerSource(llvm::StringRef Name, const char *SourceText,
60 bool IsMainFile = false) {
61 std::unique_ptr<llvm::MemoryBuffer> SourceBuf =
62 llvm::MemoryBuffer::getMemBuffer(InputData: SourceText);
63 FileEntryRef SourceFile =
64 FileMgr.getVirtualFileRef(Filename: Name, Size: SourceBuf->getBufferSize(), ModificationTime: 0);
65 SourceMgr.overrideFileContents(SourceFile, Buffer: std::move(SourceBuf));
66 FileID FID = SourceMgr.getOrCreateFileID(SourceFile, FileCharacter: SrcMgr::C_User);
67 if (IsMainFile)
68 SourceMgr.setMainFileID(FID);
69 return FID;
70 }
71
72 CharSourceRange getFakeCharSourceRange(FileID FID, LineCol Begin,
73 LineCol End) {
74 auto BeginLoc = SourceMgr.translateLineCol(FID, Line: Begin.first, Col: Begin.second);
75 auto EndLoc = SourceMgr.translateLineCol(FID, Line: End.first, Col: End.second);
76 return CharSourceRange{SourceRange{BeginLoc, EndLoc}, /* ITR = */ false};
77 }
78};
79
80TEST_F(SarifDocumentWriterTest, canCreateEmptyDocument) {
81 // GIVEN:
82 SarifDocumentWriter Writer{SourceMgr};
83
84 // WHEN:
85 const llvm::json::Object &EmptyDoc = Writer.createDocument();
86 std::vector<StringRef> Keys(EmptyDoc.size());
87 std::transform(first: EmptyDoc.begin(), last: EmptyDoc.end(), result: Keys.begin(),
88 unary_op: [](auto Item) { return Item.getFirst(); });
89
90 // THEN:
91 ASSERT_THAT(Keys, testing::UnorderedElementsAre("$schema", "version"));
92}
93
94// Test that a newly inserted run will associate correct tool names
95TEST_F(SarifDocumentWriterTest, canCreateDocumentWithOneRun) {
96 // GIVEN:
97 SarifDocumentWriter Writer{SourceMgr};
98 const char *ShortName = "sariftest";
99 const char *LongName = "sarif writer test";
100
101 // WHEN:
102 Writer.createRun(ShortToolName: ShortName, LongToolName: LongName);
103 Writer.endRun();
104 const llvm::json::Object &Doc = Writer.createDocument();
105 const llvm::json::Array *Runs = Doc.getArray(K: "runs");
106
107 // THEN:
108 // A run was created
109 ASSERT_THAT(Runs, testing::NotNull());
110
111 // It is the only run
112 ASSERT_EQ(Runs->size(), 1UL);
113
114 // The tool associated with the run was the tool
115 const llvm::json::Object *Driver =
116 Runs->begin()->getAsObject()->getObject(K: "tool")->getObject(K: "driver");
117 ASSERT_THAT(Driver, testing::NotNull());
118
119 ASSERT_TRUE(Driver->getString("name").has_value());
120 ASSERT_TRUE(Driver->getString("fullName").has_value());
121 ASSERT_TRUE(Driver->getString("language").has_value());
122
123 EXPECT_EQ(*Driver->getString("name"), ShortName);
124 EXPECT_EQ(*Driver->getString("fullName"), LongName);
125 EXPECT_EQ(*Driver->getString("language"), "en-US");
126}
127
128TEST_F(SarifDocumentWriterTest, addingResultsWillCrashIfThereIsNoRun) {
129#if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST
130 GTEST_SKIP() << "This death test is only available for debug builds.";
131#endif
132 // GIVEN:
133 SarifDocumentWriter Writer{SourceMgr};
134
135 // WHEN:
136 // A SarifDocumentWriter::createRun(...) was not called prior to
137 // SarifDocumentWriter::appendResult(...)
138 // But a rule exists
139 auto RuleIdx = Writer.createRule(Rule: SarifRule::create());
140 const SarifResult &EmptyResult = SarifResult::create(RuleIdx);
141
142 // THEN:
143 auto Matcher = ::testing::AnyOf(
144 matchers: ::testing::HasSubstr(substring: "create a run first"),
145 matchers: ::testing::HasSubstr(substring: "no runs associated with the document"));
146 ASSERT_DEATH(Writer.appendResult(EmptyResult), Matcher);
147}
148
149TEST_F(SarifDocumentWriterTest, settingInvalidRankWillCrash) {
150#if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST
151 GTEST_SKIP() << "This death test is only available for debug builds.";
152#endif
153 // GIVEN:
154 SarifDocumentWriter Writer{SourceMgr};
155
156 // WHEN:
157 // A SarifReportingConfiguration is created with an invalid "rank"
158 // * Ranks below 0.0 are invalid
159 // * Ranks above 100.0 are invalid
160
161 // THEN: The builder will crash in either case
162 EXPECT_DEATH(SarifReportingConfiguration::create().setRank(-1.0),
163 ::testing::HasSubstr("Rule rank cannot be smaller than 0.0"));
164 EXPECT_DEATH(SarifReportingConfiguration::create().setRank(101.0),
165 ::testing::HasSubstr("Rule rank cannot be larger than 100.0"));
166}
167
168TEST_F(SarifDocumentWriterTest, creatingResultWithDisabledRuleWillCrash) {
169#if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST
170 GTEST_SKIP() << "This death test is only available for debug builds.";
171#endif
172
173 // GIVEN:
174 SarifDocumentWriter Writer{SourceMgr};
175
176 // WHEN:
177 // A disabled Rule is created, and a result is create referencing this rule
178 const auto &Config = SarifReportingConfiguration::create().disable();
179 auto RuleIdx =
180 Writer.createRule(Rule: SarifRule::create().setDefaultConfiguration(Config));
181 const SarifResult &Result = SarifResult::create(RuleIdx);
182
183 // THEN:
184 // SarifResult::create(...) will produce a crash
185 ASSERT_DEATH(
186 Writer.appendResult(Result),
187 ::testing::HasSubstr("Cannot add a result referencing a disabled Rule"));
188}
189
190// Test adding rule and result shows up in the final document
191TEST_F(SarifDocumentWriterTest, addingResultWithValidRuleAndRunIsOk) {
192 // GIVEN:
193 SarifDocumentWriter Writer{SourceMgr};
194 const SarifRule &Rule =
195 SarifRule::create()
196 .setRuleId("clang.unittest")
197 .setDescription("Example rule created during unit tests")
198 .setName("clang unit test");
199
200 // WHEN:
201 Writer.createRun(ShortToolName: "sarif test", LongToolName: "sarif test runner");
202 unsigned RuleIdx = Writer.createRule(Rule);
203 const SarifResult &Result = SarifResult::create(RuleIdx);
204
205 Writer.appendResult(SarifResult: Result);
206 const llvm::json::Object &Doc = Writer.createDocument();
207
208 // THEN:
209 // A document with a valid schema and version exists
210 ASSERT_THAT(Doc.get("$schema"), ::testing::NotNull());
211 ASSERT_THAT(Doc.get("version"), ::testing::NotNull());
212 const llvm::json::Array *Runs = Doc.getArray(K: "runs");
213
214 // A run exists on this document
215 ASSERT_THAT(Runs, ::testing::NotNull());
216 ASSERT_EQ(Runs->size(), 1UL);
217 const llvm::json::Object *TheRun = Runs->back().getAsObject();
218
219 // The run has slots for tools, results, rules and artifacts
220 ASSERT_THAT(TheRun->get("tool"), ::testing::NotNull());
221 ASSERT_THAT(TheRun->get("results"), ::testing::NotNull());
222 ASSERT_THAT(TheRun->get("artifacts"), ::testing::NotNull());
223 const llvm::json::Object *Driver =
224 TheRun->getObject(K: "tool")->getObject(K: "driver");
225 const llvm::json::Array *Results = TheRun->getArray(K: "results");
226 const llvm::json::Array *Artifacts = TheRun->getArray(K: "artifacts");
227
228 // The tool is as expected
229 ASSERT_TRUE(Driver->getString("name").has_value());
230 ASSERT_TRUE(Driver->getString("fullName").has_value());
231
232 EXPECT_EQ(*Driver->getString("name"), "sarif test");
233 EXPECT_EQ(*Driver->getString("fullName"), "sarif test runner");
234
235 // The results are as expected
236 EXPECT_EQ(Results->size(), 1UL);
237
238 // The artifacts are as expected
239 EXPECT_TRUE(Artifacts->empty());
240}
241
242TEST_F(SarifDocumentWriterTest, checkSerializingResultsWithDefaultRuleConfig) {
243 // GIVEN:
244 const std::string ExpectedOutput =
245 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[],"columnKind":"unicodeCodePoints","results":[{"level":"warning","message":{"text":""},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
246
247 SarifDocumentWriter Writer{SourceMgr};
248 const SarifRule &Rule =
249 SarifRule::create()
250 .setRuleId("clang.unittest")
251 .setDescription("Example rule created during unit tests")
252 .setName("clang unit test");
253
254 // WHEN: A run contains a result
255 Writer.createRun(ShortToolName: "sarif test", LongToolName: "sarif test runner", ToolVersion: "1.0.0");
256 unsigned RuleIdx = Writer.createRule(Rule);
257 const SarifResult &Result = SarifResult::create(RuleIdx);
258 Writer.appendResult(SarifResult: Result);
259 std::string Output = serializeSarifDocument(Doc: Writer.createDocument());
260
261 // THEN:
262 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
263}
264
265TEST_F(SarifDocumentWriterTest, checkSerializingResultsWithCustomRuleConfig) {
266 // GIVEN:
267 const std::string ExpectedOutput =
268 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[],"columnKind":"unicodeCodePoints","results":[{"level":"error","message":{"text":""},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"error","rank":35.5},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
269
270 SarifDocumentWriter Writer{SourceMgr};
271 const SarifRule &Rule =
272 SarifRule::create()
273 .setRuleId("clang.unittest")
274 .setDescription("Example rule created during unit tests")
275 .setName("clang unit test")
276 .setDefaultConfiguration(SarifReportingConfiguration::create()
277 .setLevel(SarifResultLevel::Error)
278 .setRank(35.5));
279
280 // WHEN: A run contains a result
281 Writer.createRun(ShortToolName: "sarif test", LongToolName: "sarif test runner", ToolVersion: "1.0.0");
282 unsigned RuleIdx = Writer.createRule(Rule);
283 const SarifResult &Result = SarifResult::create(RuleIdx);
284 Writer.appendResult(SarifResult: Result);
285 std::string Output = serializeSarifDocument(Doc: Writer.createDocument());
286
287 // THEN:
288 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
289}
290
291// Check that serializing artifacts from results produces valid SARIF
292TEST_F(SarifDocumentWriterTest, checkSerializingArtifacts) {
293 // GIVEN:
294 const std::string ExpectedOutput =
295 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":40,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file:///main.cpp"},"region":{"endColumn":14,"startColumn":14,"startLine":3}}}],"message":{"text":"expected ';' after top level declarator"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
296
297 SarifDocumentWriter Writer{SourceMgr};
298 const SarifRule &Rule =
299 SarifRule::create()
300 .setRuleId("clang.unittest")
301 .setDescription("Example rule created during unit tests")
302 .setName("clang unit test");
303
304 // WHEN: A result is added with valid source locations for its diagnostics
305 Writer.createRun(ShortToolName: "sarif test", LongToolName: "sarif test runner", ToolVersion: "1.0.0");
306 unsigned RuleIdx = Writer.createRule(Rule);
307
308 llvm::SmallVector<CharSourceRange, 1> DiagLocs;
309 const char *SourceText = "int foo = 0;\n"
310 "int bar = 1;\n"
311 "float x = 0.0\n";
312
313 FileID MainFileID =
314 registerSource(Name: "/main.cpp", SourceText, /* IsMainFile = */ true);
315 CharSourceRange SourceCSR =
316 getFakeCharSourceRange(FID: MainFileID, Begin: {3, 14}, End: {3, 14});
317
318 DiagLocs.push_back(Elt: SourceCSR);
319
320 const SarifResult &Result =
321 SarifResult::create(RuleIdx)
322 .setLocations(DiagLocs)
323 .setDiagnosticMessage("expected ';' after top level declarator")
324 .setDiagnosticLevel(SarifResultLevel::Error);
325 Writer.appendResult(SarifResult: Result);
326 std::string Output = serializeSarifDocument(Doc: Writer.createDocument());
327
328 // THEN: Assert that the serialized SARIF is as expected
329 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
330}
331
332TEST_F(SarifDocumentWriterTest, checkSerializingCodeflows) {
333 // GIVEN:
334 const std::string ExpectedOutput =
335 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":41,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]},{"length":27,"location":{"index":1,"uri":"file:///test-header-1.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":30,"location":{"index":2,"uri":"file:///test-header-2.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":28,"location":{"index":3,"uri":"file:///test-header-3.h"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"codeFlows":[{"threadFlows":[{"locations":[{"importance":"essential","location":{"message":{"text":"Message #1"},"physicalLocation":{"artifactLocation":{"index":1,"uri":"file:///test-header-1.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"important","location":{"message":{"text":"Message #2"},"physicalLocation":{"artifactLocation":{"index":2,"uri":"file:///test-header-2.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"unimportant","location":{"message":{"text":"Message #3"},"physicalLocation":{"artifactLocation":{"index":3,"uri":"file:///test-header-3.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}}]}]}],"level":"warning","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file:///main.cpp"},"region":{"endColumn":8,"endLine":2,"startColumn":5,"startLine":2}}}],"message":{"text":"Redefinition of 'foo'"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
336
337 const char *SourceText = "int foo = 0;\n"
338 "int foo = 1;\n"
339 "float x = 0.0;\n";
340 FileID MainFileID =
341 registerSource(Name: "/main.cpp", SourceText, /* IsMainFile = */ true);
342 CharSourceRange DiagLoc{getFakeCharSourceRange(FID: MainFileID, Begin: {2, 5}, End: {2, 8})};
343
344 SarifDocumentWriter Writer{SourceMgr};
345 const SarifRule &Rule =
346 SarifRule::create()
347 .setRuleId("clang.unittest")
348 .setDescription("Example rule created during unit tests")
349 .setName("clang unit test");
350
351 constexpr unsigned int NumCases = 3;
352 llvm::SmallVector<ThreadFlow, NumCases> Threadflows;
353 const char *HeaderTexts[NumCases]{("#pragma once\n"
354 "#include <foo>"),
355 ("#ifndef FOO\n"
356 "#define FOO\n"
357 "#endif"),
358 ("#ifdef FOO\n"
359 "#undef FOO\n"
360 "#endif")};
361 const char *HeaderNames[NumCases]{"/test-header-1.h", "/test-header-2.h",
362 "/test-header-3.h"};
363 ThreadFlowImportance Importances[NumCases]{ThreadFlowImportance::Essential,
364 ThreadFlowImportance::Important,
365 ThreadFlowImportance::Unimportant};
366 for (size_t Idx = 0; Idx != NumCases; ++Idx) {
367 FileID FID = registerSource(Name: HeaderNames[Idx], SourceText: HeaderTexts[Idx]);
368 CharSourceRange &&CSR = getFakeCharSourceRange(FID, Begin: {1, 1}, End: {2, 8});
369 std::string Message = llvm::formatv(Fmt: "Message #{0}", Vals: Idx + 1);
370 ThreadFlow Item = ThreadFlow::create()
371 .setRange(CSR)
372 .setImportance(Importances[Idx])
373 .setMessage(Message);
374 Threadflows.push_back(Elt: Item);
375 }
376
377 // WHEN: A result containing code flows and diagnostic locations is added
378 Writer.createRun(ShortToolName: "sarif test", LongToolName: "sarif test runner", ToolVersion: "1.0.0");
379 unsigned RuleIdx = Writer.createRule(Rule);
380 const SarifResult &Result =
381 SarifResult::create(RuleIdx)
382 .setLocations({DiagLoc})
383 .setDiagnosticMessage("Redefinition of 'foo'")
384 .setThreadFlows(Threadflows)
385 .setDiagnosticLevel(SarifResultLevel::Warning);
386 Writer.appendResult(SarifResult: Result);
387 std::string Output = serializeSarifDocument(Doc: Writer.createDocument());
388
389 // THEN: Assert that the serialized SARIF is as expected
390 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
391}
392
393} // namespace
394

source code of clang/unittests/Basic/SarifTest.cpp