1//===- unittests/Basic/SarifTest.cpp - Test writing SARIF documents -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/Basic/Sarif.h"
10#include "clang/Basic/DiagnosticOptions.h"
11#include "clang/Basic/FileManager.h"
12#include "clang/Basic/FileSystemOptions.h"
13#include "clang/Basic/LangOptions.h"
14#include "clang/Basic/SourceLocation.h"
15#include "clang/Basic/SourceManager.h"
16#include "llvm/ADT/StringRef.h"
17#include "llvm/Support/FormatVariadic.h"
18#include "llvm/Support/JSON.h"
19#include "llvm/Support/MemoryBuffer.h"
20#include "llvm/Support/VirtualFileSystem.h"
21#include "llvm/Support/raw_ostream.h"
22#include "gmock/gmock.h"
23#include "gtest/gtest.h"
24
25#include <algorithm>
26
27using namespace clang;
28
29namespace {
30
31using LineCol = std::pair<unsigned int, unsigned int>;
32
33static std::string serializeSarifDocument(llvm::json::Object &&Doc) {
34 std::string Output;
35 llvm::json::Value Value(std::move(Doc));
36 llvm::raw_string_ostream OS{Output};
37 OS << llvm::formatv(Fmt: "{0}", Vals&: Value);
38 return Output;
39}
40
41class SarifDocumentWriterTest : public ::testing::Test {
42protected:
43 SarifDocumentWriterTest()
44 : InMemoryFileSystem(new llvm::vfs::InMemoryFileSystem),
45 FileMgr(FileSystemOptions(), InMemoryFileSystem),
46 DiagID(new DiagnosticIDs()),
47 Diags(DiagID, DiagOpts, new IgnoringDiagConsumer()),
48 SourceMgr(Diags, FileMgr) {}
49
50 IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem;
51 FileManager FileMgr;
52 IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
53 DiagnosticOptions DiagOpts;
54 DiagnosticsEngine Diags;
55 SourceManager SourceMgr;
56 LangOptions LangOpts;
57
58 FileID registerSource(llvm::StringRef Name, const char *SourceText,
59 bool IsMainFile = false) {
60 std::unique_ptr<llvm::MemoryBuffer> SourceBuf =
61 llvm::MemoryBuffer::getMemBuffer(InputData: SourceText);
62 FileEntryRef SourceFile =
63 FileMgr.getVirtualFileRef(Filename: Name, Size: SourceBuf->getBufferSize(), ModificationTime: 0);
64 SourceMgr.overrideFileContents(SourceFile, Buffer: std::move(SourceBuf));
65 FileID FID = SourceMgr.getOrCreateFileID(SourceFile, FileCharacter: SrcMgr::C_User);
66 if (IsMainFile)
67 SourceMgr.setMainFileID(FID);
68 return FID;
69 }
70
71 CharSourceRange getFakeCharSourceRange(FileID FID, LineCol Begin,
72 LineCol End) {
73 auto BeginLoc = SourceMgr.translateLineCol(FID, Line: Begin.first, Col: Begin.second);
74 auto EndLoc = SourceMgr.translateLineCol(FID, Line: End.first, Col: End.second);
75 return CharSourceRange{SourceRange{BeginLoc, EndLoc}, /* ITR = */ false};
76 }
77};
78
79TEST_F(SarifDocumentWriterTest, canCreateEmptyDocument) {
80 // GIVEN:
81 SarifDocumentWriter Writer{SourceMgr};
82
83 // WHEN:
84 const llvm::json::Object &EmptyDoc = Writer.createDocument();
85 std::vector<StringRef> Keys(EmptyDoc.size());
86 std::transform(first: EmptyDoc.begin(), last: EmptyDoc.end(), result: Keys.begin(),
87 unary_op: [](auto Item) { return Item.getFirst(); });
88
89 // THEN:
90 ASSERT_THAT(Keys, testing::UnorderedElementsAre("$schema", "version"));
91}
92
93// Test that a newly inserted run will associate correct tool names
94TEST_F(SarifDocumentWriterTest, canCreateDocumentWithOneRun) {
95 // GIVEN:
96 SarifDocumentWriter Writer{SourceMgr};
97 const char *ShortName = "sariftest";
98 const char *LongName = "sarif writer test";
99
100 // WHEN:
101 Writer.createRun(ShortToolName: ShortName, LongToolName: LongName);
102 Writer.endRun();
103 const llvm::json::Object &Doc = Writer.createDocument();
104 const llvm::json::Array *Runs = Doc.getArray(K: "runs");
105
106 // THEN:
107 // A run was created
108 ASSERT_THAT(Runs, testing::NotNull());
109
110 // It is the only run
111 ASSERT_EQ(Runs->size(), 1UL);
112
113 // The tool associated with the run was the tool
114 const llvm::json::Object *Driver =
115 Runs->begin()->getAsObject()->getObject(K: "tool")->getObject(K: "driver");
116 ASSERT_THAT(Driver, testing::NotNull());
117
118 ASSERT_TRUE(Driver->getString("name").has_value());
119 ASSERT_TRUE(Driver->getString("fullName").has_value());
120 ASSERT_TRUE(Driver->getString("language").has_value());
121
122 EXPECT_EQ(*Driver->getString("name"), ShortName);
123 EXPECT_EQ(*Driver->getString("fullName"), LongName);
124 EXPECT_EQ(*Driver->getString("language"), "en-US");
125}
126
127TEST_F(SarifDocumentWriterTest, addingResultsWillCrashIfThereIsNoRun) {
128#if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST
129 GTEST_SKIP() << "This death test is only available for debug builds.";
130#endif
131 // GIVEN:
132 SarifDocumentWriter Writer{SourceMgr};
133
134 // WHEN:
135 // A SarifDocumentWriter::createRun(...) was not called prior to
136 // SarifDocumentWriter::appendResult(...)
137 // But a rule exists
138 auto RuleIdx = Writer.createRule(Rule: SarifRule::create());
139 const SarifResult &EmptyResult = SarifResult::create(RuleIdx);
140
141 // THEN:
142 auto Matcher = ::testing::AnyOf(
143 matchers: ::testing::HasSubstr(substring: "create a run first"),
144 matchers: ::testing::HasSubstr(substring: "no runs associated with the document"));
145 ASSERT_DEATH(Writer.appendResult(EmptyResult), Matcher);
146}
147
148TEST_F(SarifDocumentWriterTest, settingInvalidRankWillCrash) {
149#if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST
150 GTEST_SKIP() << "This death test is only available for debug builds.";
151#endif
152 // GIVEN:
153 SarifDocumentWriter Writer{SourceMgr};
154
155 // WHEN:
156 // A SarifReportingConfiguration is created with an invalid "rank"
157 // * Ranks below 0.0 are invalid
158 // * Ranks above 100.0 are invalid
159
160 // THEN: The builder will crash in either case
161 EXPECT_DEATH(SarifReportingConfiguration::create().setRank(-1.0),
162 ::testing::HasSubstr("Rule rank cannot be smaller than 0.0"));
163 EXPECT_DEATH(SarifReportingConfiguration::create().setRank(101.0),
164 ::testing::HasSubstr("Rule rank cannot be larger than 100.0"));
165}
166
167TEST_F(SarifDocumentWriterTest, creatingResultWithDisabledRuleWillCrash) {
168#if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST
169 GTEST_SKIP() << "This death test is only available for debug builds.";
170#endif
171
172 // GIVEN:
173 SarifDocumentWriter Writer{SourceMgr};
174
175 // WHEN:
176 // A disabled Rule is created, and a result is create referencing this rule
177 const auto &Config = SarifReportingConfiguration::create().disable();
178 auto RuleIdx =
179 Writer.createRule(Rule: SarifRule::create().setDefaultConfiguration(Config));
180 const SarifResult &Result = SarifResult::create(RuleIdx);
181
182 // THEN:
183 // SarifResult::create(...) will produce a crash
184 ASSERT_DEATH(
185 Writer.appendResult(Result),
186 ::testing::HasSubstr("Cannot add a result referencing a disabled Rule"));
187}
188
189// Test adding rule and result shows up in the final document
190TEST_F(SarifDocumentWriterTest, addingResultWithValidRuleAndRunIsOk) {
191 // GIVEN:
192 SarifDocumentWriter Writer{SourceMgr};
193 const SarifRule &Rule =
194 SarifRule::create()
195 .setRuleId("clang.unittest")
196 .setDescription("Example rule created during unit tests")
197 .setName("clang unit test");
198
199 // WHEN:
200 Writer.createRun(ShortToolName: "sarif test", LongToolName: "sarif test runner");
201 unsigned RuleIdx = Writer.createRule(Rule);
202 const SarifResult &Result = SarifResult::create(RuleIdx);
203
204 Writer.appendResult(SarifResult: Result);
205 const llvm::json::Object &Doc = Writer.createDocument();
206
207 // THEN:
208 // A document with a valid schema and version exists
209 ASSERT_THAT(Doc.get("$schema"), ::testing::NotNull());
210 ASSERT_THAT(Doc.get("version"), ::testing::NotNull());
211 const llvm::json::Array *Runs = Doc.getArray(K: "runs");
212
213 // A run exists on this document
214 ASSERT_THAT(Runs, ::testing::NotNull());
215 ASSERT_EQ(Runs->size(), 1UL);
216 const llvm::json::Object *TheRun = Runs->back().getAsObject();
217
218 // The run has slots for tools, results, rules and artifacts
219 ASSERT_THAT(TheRun->get("tool"), ::testing::NotNull());
220 ASSERT_THAT(TheRun->get("results"), ::testing::NotNull());
221 ASSERT_THAT(TheRun->get("artifacts"), ::testing::NotNull());
222 const llvm::json::Object *Driver =
223 TheRun->getObject(K: "tool")->getObject(K: "driver");
224 const llvm::json::Array *Results = TheRun->getArray(K: "results");
225 const llvm::json::Array *Artifacts = TheRun->getArray(K: "artifacts");
226
227 // The tool is as expected
228 ASSERT_TRUE(Driver->getString("name").has_value());
229 ASSERT_TRUE(Driver->getString("fullName").has_value());
230
231 EXPECT_EQ(*Driver->getString("name"), "sarif test");
232 EXPECT_EQ(*Driver->getString("fullName"), "sarif test runner");
233
234 // The results are as expected
235 EXPECT_EQ(Results->size(), 1UL);
236
237 // The artifacts are as expected
238 EXPECT_TRUE(Artifacts->empty());
239}
240
241TEST_F(SarifDocumentWriterTest, checkSerializingResultsWithDefaultRuleConfig) {
242 // GIVEN:
243 const std::string ExpectedOutput =
244 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[],"columnKind":"unicodeCodePoints","results":[{"level":"warning","message":{"text":""},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
245
246 SarifDocumentWriter Writer{SourceMgr};
247 const SarifRule &Rule =
248 SarifRule::create()
249 .setRuleId("clang.unittest")
250 .setDescription("Example rule created during unit tests")
251 .setName("clang unit test");
252
253 // WHEN: A run contains a result
254 Writer.createRun(ShortToolName: "sarif test", LongToolName: "sarif test runner", ToolVersion: "1.0.0");
255 unsigned RuleIdx = Writer.createRule(Rule);
256 const SarifResult &Result = SarifResult::create(RuleIdx);
257 Writer.appendResult(SarifResult: Result);
258 std::string Output = serializeSarifDocument(Doc: Writer.createDocument());
259
260 // THEN:
261 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
262}
263
264TEST_F(SarifDocumentWriterTest, checkSerializingResultsWithCustomRuleConfig) {
265 // GIVEN:
266 const std::string ExpectedOutput =
267 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[],"columnKind":"unicodeCodePoints","results":[{"level":"error","message":{"text":""},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"error","rank":35.5},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
268
269 SarifDocumentWriter Writer{SourceMgr};
270 const SarifRule &Rule =
271 SarifRule::create()
272 .setRuleId("clang.unittest")
273 .setDescription("Example rule created during unit tests")
274 .setName("clang unit test")
275 .setDefaultConfiguration(SarifReportingConfiguration::create()
276 .setLevel(SarifResultLevel::Error)
277 .setRank(35.5));
278
279 // WHEN: A run contains a result
280 Writer.createRun(ShortToolName: "sarif test", LongToolName: "sarif test runner", ToolVersion: "1.0.0");
281 unsigned RuleIdx = Writer.createRule(Rule);
282 const SarifResult &Result = SarifResult::create(RuleIdx);
283 Writer.appendResult(SarifResult: Result);
284 std::string Output = serializeSarifDocument(Doc: Writer.createDocument());
285
286 // THEN:
287 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
288}
289
290// Check that serializing artifacts from results produces valid SARIF
291TEST_F(SarifDocumentWriterTest, checkSerializingArtifacts) {
292 // GIVEN:
293 const std::string ExpectedOutput =
294 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":40,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file:///main.cpp"},"region":{"endColumn":14,"startColumn":14,"startLine":3}}}],"message":{"text":"expected ';' after top level declarator"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
295
296 SarifDocumentWriter Writer{SourceMgr};
297 const SarifRule &Rule =
298 SarifRule::create()
299 .setRuleId("clang.unittest")
300 .setDescription("Example rule created during unit tests")
301 .setName("clang unit test");
302
303 // WHEN: A result is added with valid source locations for its diagnostics
304 Writer.createRun(ShortToolName: "sarif test", LongToolName: "sarif test runner", ToolVersion: "1.0.0");
305 unsigned RuleIdx = Writer.createRule(Rule);
306
307 llvm::SmallVector<CharSourceRange, 1> DiagLocs;
308 const char *SourceText = "int foo = 0;\n"
309 "int bar = 1;\n"
310 "float x = 0.0\n";
311
312 FileID MainFileID =
313 registerSource(Name: "/main.cpp", SourceText, /* IsMainFile = */ true);
314 CharSourceRange SourceCSR =
315 getFakeCharSourceRange(FID: MainFileID, Begin: {3, 14}, End: {3, 14});
316
317 DiagLocs.push_back(Elt: SourceCSR);
318
319 const SarifResult &Result =
320 SarifResult::create(RuleIdx)
321 .setLocations(DiagLocs)
322 .setDiagnosticMessage("expected ';' after top level declarator")
323 .setDiagnosticLevel(SarifResultLevel::Error);
324 Writer.appendResult(SarifResult: Result);
325 std::string Output = serializeSarifDocument(Doc: Writer.createDocument());
326
327 // THEN: Assert that the serialized SARIF is as expected
328 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
329}
330
331TEST_F(SarifDocumentWriterTest, checkSerializingCodeflows) {
332 // GIVEN:
333 const std::string ExpectedOutput =
334 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":41,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]},{"length":27,"location":{"index":1,"uri":"file:///test-header-1.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":30,"location":{"index":2,"uri":"file:///test-header-2.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":28,"location":{"index":3,"uri":"file:///test-header-3.h"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"codeFlows":[{"threadFlows":[{"locations":[{"importance":"essential","location":{"message":{"text":"Message #1"},"physicalLocation":{"artifactLocation":{"index":1,"uri":"file:///test-header-1.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"important","location":{"message":{"text":"Message #2"},"physicalLocation":{"artifactLocation":{"index":2,"uri":"file:///test-header-2.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"unimportant","location":{"message":{"text":"Message #3"},"physicalLocation":{"artifactLocation":{"index":3,"uri":"file:///test-header-3.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}}]}]}],"level":"warning","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file:///main.cpp"},"region":{"endColumn":8,"endLine":2,"startColumn":5,"startLine":2}}}],"message":{"text":"Redefinition of 'foo'"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})";
335
336 const char *SourceText = "int foo = 0;\n"
337 "int foo = 1;\n"
338 "float x = 0.0;\n";
339 FileID MainFileID =
340 registerSource(Name: "/main.cpp", SourceText, /* IsMainFile = */ true);
341 CharSourceRange DiagLoc{getFakeCharSourceRange(FID: MainFileID, Begin: {2, 5}, End: {2, 8})};
342
343 SarifDocumentWriter Writer{SourceMgr};
344 const SarifRule &Rule =
345 SarifRule::create()
346 .setRuleId("clang.unittest")
347 .setDescription("Example rule created during unit tests")
348 .setName("clang unit test");
349
350 constexpr unsigned int NumCases = 3;
351 llvm::SmallVector<ThreadFlow, NumCases> Threadflows;
352 const char *HeaderTexts[NumCases]{("#pragma once\n"
353 "#include <foo>"),
354 ("#ifndef FOO\n"
355 "#define FOO\n"
356 "#endif"),
357 ("#ifdef FOO\n"
358 "#undef FOO\n"
359 "#endif")};
360 const char *HeaderNames[NumCases]{"/test-header-1.h", "/test-header-2.h",
361 "/test-header-3.h"};
362 ThreadFlowImportance Importances[NumCases]{ThreadFlowImportance::Essential,
363 ThreadFlowImportance::Important,
364 ThreadFlowImportance::Unimportant};
365 for (size_t Idx = 0; Idx != NumCases; ++Idx) {
366 FileID FID = registerSource(Name: HeaderNames[Idx], SourceText: HeaderTexts[Idx]);
367 CharSourceRange &&CSR = getFakeCharSourceRange(FID, Begin: {1, 1}, End: {2, 8});
368 std::string Message = llvm::formatv(Fmt: "Message #{0}", Vals: Idx + 1);
369 ThreadFlow Item = ThreadFlow::create()
370 .setRange(CSR)
371 .setImportance(Importances[Idx])
372 .setMessage(Message);
373 Threadflows.push_back(Elt: Item);
374 }
375
376 // WHEN: A result containing code flows and diagnostic locations is added
377 Writer.createRun(ShortToolName: "sarif test", LongToolName: "sarif test runner", ToolVersion: "1.0.0");
378 unsigned RuleIdx = Writer.createRule(Rule);
379 const SarifResult &Result =
380 SarifResult::create(RuleIdx)
381 .setLocations({DiagLoc})
382 .setDiagnosticMessage("Redefinition of 'foo'")
383 .setThreadFlows(Threadflows)
384 .setDiagnosticLevel(SarifResultLevel::Warning);
385 Writer.appendResult(SarifResult: Result);
386 std::string Output = serializeSarifDocument(Doc: Writer.createDocument());
387
388 // THEN: Assert that the serialized SARIF is as expected
389 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput));
390}
391
392} // namespace
393

source code of clang/unittests/Basic/SarifTest.cpp