1	//===- llvm/unittest/IR/OpenMPIRBuilderTest.cpp - OpenMPIRBuilder tests ---===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "llvm/Frontend/OpenMP/OMPConstants.h"
10	#include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"
11	#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
12	#include "llvm/IR/BasicBlock.h"
13	#include "llvm/IR/DIBuilder.h"
14	#include "llvm/IR/Function.h"
15	#include "llvm/IR/InstIterator.h"
16	#include "llvm/IR/Instructions.h"
17	#include "llvm/IR/LLVMContext.h"
18	#include "llvm/IR/Module.h"
19	#include "llvm/IR/Verifier.h"
20	#include "llvm/Passes/PassBuilder.h"
21	#include "llvm/Support/Casting.h"
22	#include "llvm/Transforms/Utils/BasicBlockUtils.h"
23	#include "gmock/gmock.h"
24	#include "gtest/gtest.h"
25	#include <optional>
26
27	using namespace llvm;
28	using namespace omp;
29
30	namespace {
31
32	/// Create an instruction that uses the values in \p Values. We use "printf"
33	/// just because it is often used for this purpose in test code, but it is never
34	/// executed here.
35	static CallInst *createPrintfCall(IRBuilder<> &Builder, StringRef FormatStr,
36	ArrayRef<Value *> Values) {
37	Module *M = Builder.GetInsertBlock()->getParent()->getParent();
38
39	GlobalVariable *GV = Builder.CreateGlobalString(Str: FormatStr, Name: "", AddressSpace: 0, M);
40	Constant *Zero = ConstantInt::get(Ty: Type::getInt32Ty(C&: M->getContext()), V: 0);
41	Constant *Indices[] = {Zero, Zero};
42	Constant *FormatStrConst =
43	ConstantExpr::getInBoundsGetElementPtr(Ty: GV->getValueType(), C: GV, IdxList: Indices);
44
45	Function *PrintfDecl = M->getFunction(Name: "printf");
46	if (!PrintfDecl) {
47	GlobalValue::LinkageTypes Linkage = Function::ExternalLinkage;
48	FunctionType *Ty = FunctionType::get(Result: Builder.getInt32Ty(), isVarArg: true);
49	PrintfDecl = Function::Create(Ty, Linkage, N: "printf", M);
50	}
51
52	SmallVector<Value *, 4> Args;
53	Args.push_back(Elt: FormatStrConst);
54	Args.append(in_start: Values.begin(), in_end: Values.end());
55	return Builder.CreateCall(Callee: PrintfDecl, Args);
56	}
57
58	/// Verify that blocks in \p RefOrder are corresponds to the depth-first visit
59	/// order the control flow of \p F.
60	///
61	/// This is an easy way to verify the branching structure of the CFG without
62	/// checking every branch instruction individually. For the CFG of a
63	/// CanonicalLoopInfo, the Cond BB's terminating branch's first edge is entering
64	/// the body, i.e. the DFS order corresponds to the execution order with one
65	/// loop iteration.
66	static testing::AssertionResult
67	verifyDFSOrder(Function *F, ArrayRef<BasicBlock *> RefOrder) {
68	ArrayRef<BasicBlock *>::iterator It = RefOrder.begin();
69	ArrayRef<BasicBlock *>::iterator E = RefOrder.end();
70
71	df_iterator_default_set<BasicBlock *, 16> Visited;
72	auto DFS = llvm::depth_first_ext(G: &F->getEntryBlock(), S&: Visited);
73
74	BasicBlock *Prev = nullptr;
75	for (BasicBlock *BB : DFS) {
76	if (It != E && BB == *It) {
77	Prev = *It;
78	++It;
79	}
80	}
81
82	if (It == E)
83	return testing::AssertionSuccess();
84	if (!Prev)
85	return testing::AssertionFailure()
86	<< "Did not find " << (*It)->getName() << " in control flow";
87	return testing::AssertionFailure()
88	<< "Expected " << Prev->getName() << " before " << (*It)->getName()
89	<< " in control flow";
90	}
91
92	/// Verify that blocks in \p RefOrder are in the same relative order in the
93	/// linked lists of blocks in \p F. The linked list may contain additional
94	/// blocks in-between.
95	///
96	/// While the order in the linked list is not relevant for semantics, keeping
97	/// the order roughly in execution order makes its printout easier to read.
98	static testing::AssertionResult
99	verifyListOrder(Function *F, ArrayRef<BasicBlock *> RefOrder) {
100	ArrayRef<BasicBlock *>::iterator It = RefOrder.begin();
101	ArrayRef<BasicBlock *>::iterator E = RefOrder.end();
102
103	BasicBlock *Prev = nullptr;
104	for (BasicBlock &BB : *F) {
105	if (It != E && &BB == *It) {
106	Prev = *It;
107	++It;
108	}
109	}
110
111	if (It == E)
112	return testing::AssertionSuccess();
113	if (!Prev)
114	return testing::AssertionFailure() << "Did not find " << (*It)->getName()
115	<< " in function " << F->getName();
116	return testing::AssertionFailure()
117	<< "Expected " << Prev->getName() << " before " << (*It)->getName()
118	<< " in function " << F->getName();
119	}
120
121	/// Populate Calls with call instructions calling the function with the given
122	/// FnID from the given function F.
123	static void findCalls(Function *F, omp::RuntimeFunction FnID,
124	OpenMPIRBuilder &OMPBuilder,
125	SmallVectorImpl<CallInst *> &Calls) {
126	Function *Fn = OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID);
127	for (BasicBlock &BB : *F) {
128	for (Instruction &I : BB) {
129	auto *Call = dyn_cast<CallInst>(Val: &I);
130	if (Call && Call->getCalledFunction() == Fn)
131	Calls.push_back(Elt: Call);
132	}
133	}
134	}
135
136	/// Assuming \p F contains only one call to the function with the given \p FnID,
137	/// return that call.
138	static CallInst *findSingleCall(Function *F, omp::RuntimeFunction FnID,
139	OpenMPIRBuilder &OMPBuilder) {
140	SmallVector<CallInst *, 1> Calls;
141	findCalls(F, FnID, OMPBuilder, Calls);
142	EXPECT_EQ(1u, Calls.size());
143	if (Calls.size() != 1)
144	return nullptr;
145	return Calls.front();
146	}
147
148	static omp::ScheduleKind getSchedKind(omp::OMPScheduleType SchedType) {
149	switch (SchedType & ~omp::OMPScheduleType::ModifierMask) {
150	case omp::OMPScheduleType::BaseDynamicChunked:
151	return omp::OMP_SCHEDULE_Dynamic;
152	case omp::OMPScheduleType::BaseGuidedChunked:
153	return omp::OMP_SCHEDULE_Guided;
154	case omp::OMPScheduleType::BaseAuto:
155	return omp::OMP_SCHEDULE_Auto;
156	case omp::OMPScheduleType::BaseRuntime:
157	return omp::OMP_SCHEDULE_Runtime;
158	default:
159	llvm_unreachable("unknown type for this test");
160	}
161	}
162
163	class OpenMPIRBuilderTest : public testing::Test {
164	protected:
165	void SetUp() override {
166	M.reset(p: new Module("MyModule", Ctx));
167	FunctionType *FTy =
168	FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: {Type::getInt32Ty(C&: Ctx)},
169	/*isVarArg=*/false);
170	F = Function::Create(Ty: FTy, Linkage: Function::ExternalLinkage, N: "", M: M.get());
171	BB = BasicBlock::Create(Context&: Ctx, Name: "", Parent: F);
172
173	DIBuilder DIB(*M);
174	auto File = DIB.createFile(Filename: "test.dbg", Directory: "/src", Checksum: std::nullopt,
175	Source: std::optional<StringRef>("/src/test.dbg"));
176	auto CU =
177	DIB.createCompileUnit(Lang: dwarf::DW_LANG_C, File, Producer: "llvm-C", isOptimized: true, Flags: "", RV: 0);
178	auto Type =
179	DIB.createSubroutineType(ParameterTypes: DIB.getOrCreateTypeArray(Elements: std::nullopt));
180	auto SP = DIB.createFunction(
181	Scope: CU, Name: "foo", LinkageName: "", File, LineNo: 1, Ty: Type, ScopeLine: 1, Flags: DINode::FlagZero,
182	SPFlags: DISubprogram::SPFlagDefinition | DISubprogram::SPFlagOptimized);
183	F->setSubprogram(SP);
184	auto Scope = DIB.createLexicalBlockFile(Scope: SP, File, Discriminator: 0);
185	DIB.finalize();
186	DL = DILocation::get(Context&: Ctx, Line: 3, Column: 7, Scope);
187	}
188
189	void TearDown() override {
190	BB = nullptr;
191	M.reset();
192	}
193
194	/// Create a function with a simple loop that calls printf using the logical
195	/// loop counter for use with tests that need a CanonicalLoopInfo object.
196	CanonicalLoopInfo *buildSingleLoopFunction(DebugLoc DL,
197	OpenMPIRBuilder &OMPBuilder,
198	int UseIVBits,
199	CallInst **Call = nullptr,
200	BasicBlock **BodyCode = nullptr) {
201	OMPBuilder.initialize();
202	F->setName("func");
203
204	IRBuilder<> Builder(BB);
205	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
206	Value *TripCount = F->getArg(i: 0);
207
208	Type *IVType = Type::getIntNTy(C&: Builder.getContext(), N: UseIVBits);
209	Value *CastedTripCount =
210	Builder.CreateZExtOrTrunc(V: TripCount, DestTy: IVType, Name: "tripcount");
211
212	auto LoopBodyGenCB = [&](OpenMPIRBuilder::InsertPointTy CodeGenIP,
213	llvm::Value *LC) {
214	Builder.restoreIP(IP: CodeGenIP);
215	if (BodyCode)
216	*BodyCode = Builder.GetInsertBlock();
217
218	// Add something that consumes the induction variable to the body.
219	CallInst *CallInst = createPrintfCall(Builder, FormatStr: "%d\\n", Values: {LC});
220	if (Call)
221	*Call = CallInst;
222	};
223	CanonicalLoopInfo *Loop =
224	OMPBuilder.createCanonicalLoop(Loc, BodyGenCB: LoopBodyGenCB, TripCount: CastedTripCount);
225
226	// Finalize the function.
227	Builder.restoreIP(IP: Loop->getAfterIP());
228	Builder.CreateRetVoid();
229
230	return Loop;
231	}
232
233	LLVMContext Ctx;
234	std::unique_ptr<Module> M;
235	Function *F;
236	BasicBlock *BB;
237	DebugLoc DL;
238	};
239
240	class OpenMPIRBuilderTestWithParams
241	: public OpenMPIRBuilderTest,
242	public ::testing::WithParamInterface<omp::OMPScheduleType> {};
243
244	class OpenMPIRBuilderTestWithIVBits
245	: public OpenMPIRBuilderTest,
246	public ::testing::WithParamInterface<int> {};
247
248	// Returns the value stored in the given allocation. Returns null if the given
249	// value is not a result of an InstTy instruction, if no value is stored or if
250	// there is more than one store.
251	template <typename InstTy> static Value *findStoredValue(Value *AllocaValue) {
252	Instruction *Inst = dyn_cast<InstTy>(AllocaValue);
253	if (!Inst)
254	return nullptr;
255	StoreInst *Store = nullptr;
256	for (Use &U : Inst->uses()) {
257	if (auto *CandidateStore = dyn_cast<StoreInst>(Val: U.getUser())) {
258	EXPECT_EQ(Store, nullptr);
259	Store = CandidateStore;
260	}
261	}
262	if (!Store)
263	return nullptr;
264	return Store->getValueOperand();
265	}
266
267	// Returns the value stored in the aggregate argument of an outlined function,
268	// or nullptr if it is not found.
269	static Value *findStoredValueInAggregateAt(LLVMContext &Ctx, Value *Aggregate,
270	unsigned Idx) {
271	GetElementPtrInst *GEPAtIdx = nullptr;
272	// Find GEP instruction at that index.
273	for (User *Usr : Aggregate->users()) {
274	GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Usr);
275	if (!GEP)
276	continue;
277
278	if (GEP->getOperand(i_nocapture: 2) != ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: Idx))
279	continue;
280
281	EXPECT_EQ(GEPAtIdx, nullptr);
282	GEPAtIdx = GEP;
283	}
284
285	EXPECT_NE(GEPAtIdx, nullptr);
286	EXPECT_EQ(GEPAtIdx->getNumUses(), 1U);
287
288	// Find the value stored to the aggregate.
289	StoreInst *StoreToAgg = dyn_cast<StoreInst>(Val: *GEPAtIdx->user_begin());
290	Value *StoredAggValue = StoreToAgg->getValueOperand();
291
292	Value *StoredValue = nullptr;
293
294	// Find the value stored to the value stored in the aggregate.
295	for (User *Usr : StoredAggValue->users()) {
296	StoreInst *Store = dyn_cast<StoreInst>(Val: Usr);
297	if (!Store)
298	continue;
299
300	if (Store->getPointerOperand() != StoredAggValue)
301	continue;
302
303	EXPECT_EQ(StoredValue, nullptr);
304	StoredValue = Store->getValueOperand();
305	}
306
307	return StoredValue;
308	}
309
310	// Returns the aggregate that the value is originating from.
311	static Value *findAggregateFromValue(Value *V) {
312	// Expects a load instruction that loads from the aggregate.
313	LoadInst *Load = dyn_cast<LoadInst>(Val: V);
314	EXPECT_NE(Load, nullptr);
315	// Find the GEP instruction used in the load instruction.
316	GetElementPtrInst *GEP =
317	dyn_cast<GetElementPtrInst>(Val: Load->getPointerOperand());
318	EXPECT_NE(GEP, nullptr);
319	// Find the aggregate used in the GEP instruction.
320	Value *Aggregate = GEP->getPointerOperand();
321
322	return Aggregate;
323	}
324
325	TEST_F(OpenMPIRBuilderTest, CreateBarrier) {
326	OpenMPIRBuilder OMPBuilder(*M);
327	OMPBuilder.initialize();
328
329	IRBuilder<> Builder(BB);
330
331	OMPBuilder.createBarrier({IRBuilder<>::InsertPoint()}, OMPD_for);
332	EXPECT_TRUE(M->global_empty());
333	EXPECT_EQ(M->size(), 1U);
334	EXPECT_EQ(F->size(), 1U);
335	EXPECT_EQ(BB->size(), 0U);
336
337	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
338	OMPBuilder.createBarrier(Loc, OMPD_for);
339	EXPECT_FALSE(M->global_empty());
340	EXPECT_EQ(M->size(), 3U);
341	EXPECT_EQ(F->size(), 1U);
342	EXPECT_EQ(BB->size(), 2U);
343
344	CallInst *GTID = dyn_cast<CallInst>(Val: &BB->front());
345	EXPECT_NE(GTID, nullptr);
346	EXPECT_EQ(GTID->arg_size(), 1U);
347	EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
348	EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
349	EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
350
351	CallInst *Barrier = dyn_cast<CallInst>(Val: GTID->getNextNode());
352	EXPECT_NE(Barrier, nullptr);
353	EXPECT_EQ(Barrier->arg_size(), 2U);
354	EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_barrier");
355	EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
356	EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
357
358	EXPECT_EQ(cast<CallInst>(Barrier)->getArgOperand(1), GTID);
359
360	Builder.CreateUnreachable();
361	EXPECT_FALSE(verifyModule(*M, &errs()));
362	}
363
364	TEST_F(OpenMPIRBuilderTest, CreateCancel) {
365	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
366	OpenMPIRBuilder OMPBuilder(*M);
367	OMPBuilder.initialize();
368
369	BasicBlock *CBB = BasicBlock::Create(Context&: Ctx, Name: "", Parent: F);
370	new UnreachableInst(Ctx, CBB);
371	auto FiniCB = [&](InsertPointTy IP) {
372	ASSERT_NE(IP.getBlock(), nullptr);
373	ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
374	BranchInst::Create(IfTrue: CBB, InsertAtEnd: IP.getBlock());
375	};
376	OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
377
378	IRBuilder<> Builder(BB);
379
380	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
381	auto NewIP = OMPBuilder.createCancel(Loc, nullptr, OMPD_parallel);
382	Builder.restoreIP(IP: NewIP);
383	EXPECT_FALSE(M->global_empty());
384	EXPECT_EQ(M->size(), 4U);
385	EXPECT_EQ(F->size(), 4U);
386	EXPECT_EQ(BB->size(), 4U);
387
388	CallInst *GTID = dyn_cast<CallInst>(Val: &BB->front());
389	EXPECT_NE(GTID, nullptr);
390	EXPECT_EQ(GTID->arg_size(), 1U);
391	EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
392	EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
393	EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
394
395	CallInst *Cancel = dyn_cast<CallInst>(Val: GTID->getNextNode());
396	EXPECT_NE(Cancel, nullptr);
397	EXPECT_EQ(Cancel->arg_size(), 3U);
398	EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel");
399	EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory());
400	EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory());
401	EXPECT_EQ(Cancel->getNumUses(), 1U);
402	Instruction *CancelBBTI = Cancel->getParent()->getTerminator();
403	EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);
404	EXPECT_EQ(CancelBBTI->getSuccessor(0), NewIP.getBlock());
405	EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U);
406	CallInst *GTID1 = dyn_cast<CallInst>(Val: &CancelBBTI->getSuccessor(Idx: 1)->front());
407	EXPECT_NE(GTID1, nullptr);
408	EXPECT_EQ(GTID1->arg_size(), 1U);
409	EXPECT_EQ(GTID1->getCalledFunction()->getName(), "__kmpc_global_thread_num");
410	EXPECT_FALSE(GTID1->getCalledFunction()->doesNotAccessMemory());
411	EXPECT_FALSE(GTID1->getCalledFunction()->doesNotFreeMemory());
412	CallInst *Barrier = dyn_cast<CallInst>(Val: GTID1->getNextNode());
413	EXPECT_NE(Barrier, nullptr);
414	EXPECT_EQ(Barrier->arg_size(), 2U);
415	EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
416	EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
417	EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
418	EXPECT_EQ(Barrier->getNumUses(), 0U);
419	EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
420	1U);
421	EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB);
422
423	EXPECT_EQ(cast<CallInst>(Cancel)->getArgOperand(1), GTID);
424
425	OMPBuilder.popFinalizationCB();
426
427	Builder.CreateUnreachable();
428	EXPECT_FALSE(verifyModule(*M, &errs()));
429	}
430
431	TEST_F(OpenMPIRBuilderTest, CreateCancelIfCond) {
432	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
433	OpenMPIRBuilder OMPBuilder(*M);
434	OMPBuilder.initialize();
435
436	BasicBlock *CBB = BasicBlock::Create(Context&: Ctx, Name: "", Parent: F);
437	new UnreachableInst(Ctx, CBB);
438	auto FiniCB = [&](InsertPointTy IP) {
439	ASSERT_NE(IP.getBlock(), nullptr);
440	ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
441	BranchInst::Create(IfTrue: CBB, InsertAtEnd: IP.getBlock());
442	};
443	OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
444
445	IRBuilder<> Builder(BB);
446
447	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
448	auto NewIP = OMPBuilder.createCancel(Loc, Builder.getTrue(), OMPD_parallel);
449	Builder.restoreIP(IP: NewIP);
450	EXPECT_FALSE(M->global_empty());
451	EXPECT_EQ(M->size(), 4U);
452	EXPECT_EQ(F->size(), 7U);
453	EXPECT_EQ(BB->size(), 1U);
454	ASSERT_TRUE(isa<BranchInst>(BB->getTerminator()));
455	ASSERT_EQ(BB->getTerminator()->getNumSuccessors(), 2U);
456	BB = BB->getTerminator()->getSuccessor(Idx: 0);
457	EXPECT_EQ(BB->size(), 4U);
458
459	CallInst *GTID = dyn_cast<CallInst>(Val: &BB->front());
460	EXPECT_NE(GTID, nullptr);
461	EXPECT_EQ(GTID->arg_size(), 1U);
462	EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
463	EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
464	EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
465
466	CallInst *Cancel = dyn_cast<CallInst>(Val: GTID->getNextNode());
467	EXPECT_NE(Cancel, nullptr);
468	EXPECT_EQ(Cancel->arg_size(), 3U);
469	EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel");
470	EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory());
471	EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory());
472	EXPECT_EQ(Cancel->getNumUses(), 1U);
473	Instruction *CancelBBTI = Cancel->getParent()->getTerminator();
474	EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);
475	EXPECT_EQ(CancelBBTI->getSuccessor(0)->size(), 1U);
476	EXPECT_EQ(CancelBBTI->getSuccessor(0)->getUniqueSuccessor(),
477	NewIP.getBlock());
478	EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U);
479	CallInst *GTID1 = dyn_cast<CallInst>(Val: &CancelBBTI->getSuccessor(Idx: 1)->front());
480	EXPECT_NE(GTID1, nullptr);
481	EXPECT_EQ(GTID1->arg_size(), 1U);
482	EXPECT_EQ(GTID1->getCalledFunction()->getName(), "__kmpc_global_thread_num");
483	EXPECT_FALSE(GTID1->getCalledFunction()->doesNotAccessMemory());
484	EXPECT_FALSE(GTID1->getCalledFunction()->doesNotFreeMemory());
485	CallInst *Barrier = dyn_cast<CallInst>(Val: GTID1->getNextNode());
486	EXPECT_NE(Barrier, nullptr);
487	EXPECT_EQ(Barrier->arg_size(), 2U);
488	EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
489	EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
490	EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
491	EXPECT_EQ(Barrier->getNumUses(), 0U);
492	EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
493	1U);
494	EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB);
495
496	EXPECT_EQ(cast<CallInst>(Cancel)->getArgOperand(1), GTID);
497
498	OMPBuilder.popFinalizationCB();
499
500	Builder.CreateUnreachable();
501	EXPECT_FALSE(verifyModule(*M, &errs()));
502	}
503
504	TEST_F(OpenMPIRBuilderTest, CreateCancelBarrier) {
505	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
506	OpenMPIRBuilder OMPBuilder(*M);
507	OMPBuilder.initialize();
508
509	BasicBlock *CBB = BasicBlock::Create(Context&: Ctx, Name: "", Parent: F);
510	new UnreachableInst(Ctx, CBB);
511	auto FiniCB = [&](InsertPointTy IP) {
512	ASSERT_NE(IP.getBlock(), nullptr);
513	ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
514	BranchInst::Create(IfTrue: CBB, InsertAtEnd: IP.getBlock());
515	};
516	OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
517
518	IRBuilder<> Builder(BB);
519
520	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
521	auto NewIP = OMPBuilder.createBarrier(Loc, OMPD_for);
522	Builder.restoreIP(IP: NewIP);
523	EXPECT_FALSE(M->global_empty());
524	EXPECT_EQ(M->size(), 3U);
525	EXPECT_EQ(F->size(), 4U);
526	EXPECT_EQ(BB->size(), 4U);
527
528	CallInst *GTID = dyn_cast<CallInst>(Val: &BB->front());
529	EXPECT_NE(GTID, nullptr);
530	EXPECT_EQ(GTID->arg_size(), 1U);
531	EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
532	EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
533	EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
534
535	CallInst *Barrier = dyn_cast<CallInst>(Val: GTID->getNextNode());
536	EXPECT_NE(Barrier, nullptr);
537	EXPECT_EQ(Barrier->arg_size(), 2U);
538	EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
539	EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
540	EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
541	EXPECT_EQ(Barrier->getNumUses(), 1U);
542	Instruction *BarrierBBTI = Barrier->getParent()->getTerminator();
543	EXPECT_EQ(BarrierBBTI->getNumSuccessors(), 2U);
544	EXPECT_EQ(BarrierBBTI->getSuccessor(0), NewIP.getBlock());
545	EXPECT_EQ(BarrierBBTI->getSuccessor(1)->size(), 1U);
546	EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
547	1U);
548	EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0),
549	CBB);
550
551	EXPECT_EQ(cast<CallInst>(Barrier)->getArgOperand(1), GTID);
552
553	OMPBuilder.popFinalizationCB();
554
555	Builder.CreateUnreachable();
556	EXPECT_FALSE(verifyModule(*M, &errs()));
557	}
558
559	TEST_F(OpenMPIRBuilderTest, DbgLoc) {
560	OpenMPIRBuilder OMPBuilder(*M);
561	OMPBuilder.initialize();
562	F->setName("func");
563
564	IRBuilder<> Builder(BB);
565
566	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
567	OMPBuilder.createBarrier(Loc, OMPD_for);
568	CallInst *GTID = dyn_cast<CallInst>(Val: &BB->front());
569	CallInst *Barrier = dyn_cast<CallInst>(Val: GTID->getNextNode());
570	EXPECT_EQ(GTID->getDebugLoc(), DL);
571	EXPECT_EQ(Barrier->getDebugLoc(), DL);
572	EXPECT_TRUE(isa<GlobalVariable>(Barrier->getOperand(0)));
573	if (!isa<GlobalVariable>(Val: Barrier->getOperand(i_nocapture: 0)))
574	return;
575	GlobalVariable *Ident = cast<GlobalVariable>(Val: Barrier->getOperand(i_nocapture: 0));
576	EXPECT_TRUE(Ident->hasInitializer());
577	if (!Ident->hasInitializer())
578	return;
579	Constant *Initializer = Ident->getInitializer();
580	EXPECT_TRUE(
581	isa<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts()));
582	GlobalVariable *SrcStrGlob =
583	cast<GlobalVariable>(Val: Initializer->getOperand(i: 4)->stripPointerCasts());
584	if (!SrcStrGlob)
585	return;
586	EXPECT_TRUE(isa<ConstantDataArray>(SrcStrGlob->getInitializer()));
587	ConstantDataArray *SrcSrc =
588	dyn_cast<ConstantDataArray>(Val: SrcStrGlob->getInitializer());
589	if (!SrcSrc)
590	return;
591	EXPECT_EQ(SrcSrc->getAsCString(), ";/src/test.dbg;foo;3;7;;");
592	}
593
594	TEST_F(OpenMPIRBuilderTest, ParallelSimpleGPU) {
595	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
596	std::string oldDLStr = M->getDataLayoutStr();
597	M->setDataLayout(
598	"e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:"
599	"256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:"
600	"256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8");
601	OpenMPIRBuilder OMPBuilder(*M);
602	OMPBuilder.Config.IsTargetDevice = true;
603	OMPBuilder.initialize();
604	F->setName("func");
605	IRBuilder<> Builder(BB);
606	BasicBlock *EnterBB = BasicBlock::Create(Context&: Ctx, Name: "parallel.enter", Parent: F);
607	Builder.CreateBr(Dest: EnterBB);
608	Builder.SetInsertPoint(EnterBB);
609	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
610
611	AllocaInst *PrivAI = nullptr;
612
613	unsigned NumBodiesGenerated = 0;
614	unsigned NumPrivatizedVars = 0;
615	unsigned NumFinalizationPoints = 0;
616
617	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
618	++NumBodiesGenerated;
619
620	Builder.restoreIP(IP: AllocaIP);
621	PrivAI = Builder.CreateAlloca(Ty: F->arg_begin()->getType());
622	Builder.CreateStore(Val: F->arg_begin(), Ptr: PrivAI);
623
624	Builder.restoreIP(IP: CodeGenIP);
625	Value *PrivLoad =
626	Builder.CreateLoad(Ty: PrivAI->getAllocatedType(), Ptr: PrivAI, Name: "local.use");
627	Value *Cmp = Builder.CreateICmpNE(LHS: F->arg_begin(), RHS: PrivLoad);
628	Instruction *ThenTerm, *ElseTerm;
629	SplitBlockAndInsertIfThenElse(Cond: Cmp, SplitBefore: CodeGenIP.getBlock()->getTerminator(),
630	ThenTerm: &ThenTerm, ElseTerm: &ElseTerm);
631	};
632
633	auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
634	Value &Orig, Value &Inner,
635	Value *&ReplacementValue) -> InsertPointTy {
636	++NumPrivatizedVars;
637
638	if (!isa<AllocaInst>(Val: Orig)) {
639	EXPECT_EQ(&Orig, F->arg_begin());
640	ReplacementValue = &Inner;
641	return CodeGenIP;
642	}
643
644	// Since the original value is an allocation, it has a pointer type and
645	// therefore no additional wrapping should happen.
646	EXPECT_EQ(&Orig, &Inner);
647
648	// Trivial copy (=firstprivate).
649	Builder.restoreIP(IP: AllocaIP);
650	Type *VTy = ReplacementValue->getType();
651	Value *V = Builder.CreateLoad(Ty: VTy, Ptr: &Inner, Name: Orig.getName() + ".reload");
652	ReplacementValue = Builder.CreateAlloca(Ty: VTy, ArraySize: 0, Name: Orig.getName() + ".copy");
653	Builder.restoreIP(IP: CodeGenIP);
654	Builder.CreateStore(Val: V, Ptr: ReplacementValue);
655	return CodeGenIP;
656	};
657
658	auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
659
660	IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
661	F->getEntryBlock().getFirstInsertionPt());
662	IRBuilder<>::InsertPoint AfterIP =
663	OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
664	nullptr, nullptr, OMP_PROC_BIND_default, false);
665
666	EXPECT_EQ(NumBodiesGenerated, 1U);
667	EXPECT_EQ(NumPrivatizedVars, 1U);
668	EXPECT_EQ(NumFinalizationPoints, 1U);
669
670	Builder.restoreIP(IP: AfterIP);
671	Builder.CreateRetVoid();
672
673	OMPBuilder.finalize();
674	Function *OutlinedFn = PrivAI->getFunction();
675	EXPECT_FALSE(verifyModule(*M, &errs()));
676	EXPECT_NE(OutlinedFn, F);
677	EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoUnwind));
678	EXPECT_TRUE(OutlinedFn->hasParamAttribute(0, Attribute::NoAlias));
679	EXPECT_TRUE(OutlinedFn->hasParamAttribute(1, Attribute::NoAlias));
680
681	EXPECT_TRUE(OutlinedFn->hasInternalLinkage());
682	EXPECT_EQ(OutlinedFn->arg_size(), 3U);
683	// Make sure that arguments are pointers in 0 address address space
684	EXPECT_EQ(OutlinedFn->getArg(0)->getType(),
685	PointerType::get(M->getContext(), 0));
686	EXPECT_EQ(OutlinedFn->getArg(1)->getType(),
687	PointerType::get(M->getContext(), 0));
688	EXPECT_EQ(OutlinedFn->getArg(2)->getType(),
689	PointerType::get(M->getContext(), 0));
690	EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());
691	EXPECT_EQ(OutlinedFn->getNumUses(), 1U);
692	User *Usr = OutlinedFn->user_back();
693	ASSERT_TRUE(isa<CallInst>(Usr));
694	CallInst *Parallel51CI = dyn_cast<CallInst>(Val: Usr);
695	ASSERT_NE(Parallel51CI, nullptr);
696
697	EXPECT_EQ(Parallel51CI->getCalledFunction()->getName(), "__kmpc_parallel_51");
698	EXPECT_EQ(Parallel51CI->arg_size(), 9U);
699	EXPECT_EQ(Parallel51CI->getArgOperand(5), OutlinedFn);
700	EXPECT_TRUE(
701	isa<GlobalVariable>(Parallel51CI->getArgOperand(0)->stripPointerCasts()));
702	EXPECT_EQ(Parallel51CI, Usr);
703	M->setDataLayout(oldDLStr);
704	}
705
706	TEST_F(OpenMPIRBuilderTest, ParallelSimple) {
707	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
708	OpenMPIRBuilder OMPBuilder(*M);
709	OMPBuilder.Config.IsTargetDevice = false;
710	OMPBuilder.initialize();
711	F->setName("func");
712	IRBuilder<> Builder(BB);
713
714	BasicBlock *EnterBB = BasicBlock::Create(Context&: Ctx, Name: "parallel.enter", Parent: F);
715	Builder.CreateBr(Dest: EnterBB);
716	Builder.SetInsertPoint(EnterBB);
717	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
718
719	AllocaInst *PrivAI = nullptr;
720
721	unsigned NumBodiesGenerated = 0;
722	unsigned NumPrivatizedVars = 0;
723	unsigned NumFinalizationPoints = 0;
724
725	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
726	++NumBodiesGenerated;
727
728	Builder.restoreIP(IP: AllocaIP);
729	PrivAI = Builder.CreateAlloca(Ty: F->arg_begin()->getType());
730	Builder.CreateStore(Val: F->arg_begin(), Ptr: PrivAI);
731
732	Builder.restoreIP(IP: CodeGenIP);
733	Value *PrivLoad =
734	Builder.CreateLoad(Ty: PrivAI->getAllocatedType(), Ptr: PrivAI, Name: "local.use");
735	Value *Cmp = Builder.CreateICmpNE(LHS: F->arg_begin(), RHS: PrivLoad);
736	Instruction *ThenTerm, *ElseTerm;
737	SplitBlockAndInsertIfThenElse(Cond: Cmp, SplitBefore: CodeGenIP.getBlock()->getTerminator(),
738	ThenTerm: &ThenTerm, ElseTerm: &ElseTerm);
739	};
740
741	auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
742	Value &Orig, Value &Inner,
743	Value *&ReplacementValue) -> InsertPointTy {
744	++NumPrivatizedVars;
745
746	if (!isa<AllocaInst>(Val: Orig)) {
747	EXPECT_EQ(&Orig, F->arg_begin());
748	ReplacementValue = &Inner;
749	return CodeGenIP;
750	}
751
752	// Since the original value is an allocation, it has a pointer type and
753	// therefore no additional wrapping should happen.
754	EXPECT_EQ(&Orig, &Inner);
755
756	// Trivial copy (=firstprivate).
757	Builder.restoreIP(IP: AllocaIP);
758	Type *VTy = ReplacementValue->getType();
759	Value *V = Builder.CreateLoad(Ty: VTy, Ptr: &Inner, Name: Orig.getName() + ".reload");
760	ReplacementValue = Builder.CreateAlloca(Ty: VTy, ArraySize: 0, Name: Orig.getName() + ".copy");
761	Builder.restoreIP(IP: CodeGenIP);
762	Builder.CreateStore(Val: V, Ptr: ReplacementValue);
763	return CodeGenIP;
764	};
765
766	auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
767
768	IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
769	F->getEntryBlock().getFirstInsertionPt());
770	IRBuilder<>::InsertPoint AfterIP =
771	OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
772	nullptr, nullptr, OMP_PROC_BIND_default, false);
773	EXPECT_EQ(NumBodiesGenerated, 1U);
774	EXPECT_EQ(NumPrivatizedVars, 1U);
775	EXPECT_EQ(NumFinalizationPoints, 1U);
776
777	Builder.restoreIP(IP: AfterIP);
778	Builder.CreateRetVoid();
779
780	OMPBuilder.finalize();
781
782	EXPECT_NE(PrivAI, nullptr);
783	Function *OutlinedFn = PrivAI->getFunction();
784	EXPECT_NE(F, OutlinedFn);
785	EXPECT_FALSE(verifyModule(*M, &errs()));
786	EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoUnwind));
787	EXPECT_TRUE(OutlinedFn->hasParamAttribute(0, Attribute::NoAlias));
788	EXPECT_TRUE(OutlinedFn->hasParamAttribute(1, Attribute::NoAlias));
789
790	EXPECT_TRUE(OutlinedFn->hasInternalLinkage());
791	EXPECT_EQ(OutlinedFn->arg_size(), 3U);
792
793	EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());
794	EXPECT_EQ(OutlinedFn->getNumUses(), 1U);
795	User *Usr = OutlinedFn->user_back();
796	ASSERT_TRUE(isa<CallInst>(Usr));
797	CallInst *ForkCI = dyn_cast<CallInst>(Val: Usr);
798	ASSERT_NE(ForkCI, nullptr);
799
800	EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
801	EXPECT_EQ(ForkCI->arg_size(), 4U);
802	EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
803	EXPECT_EQ(ForkCI->getArgOperand(1),
804	ConstantInt::get(Type::getInt32Ty(Ctx), 1U));
805	EXPECT_EQ(ForkCI, Usr);
806	Value *StoredValue =
807	findStoredValueInAggregateAt(Ctx, Aggregate: ForkCI->getArgOperand(i: 3), Idx: 0);
808	EXPECT_EQ(StoredValue, F->arg_begin());
809	}
810
811	TEST_F(OpenMPIRBuilderTest, ParallelNested) {
812	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
813	OpenMPIRBuilder OMPBuilder(*M);
814	OMPBuilder.Config.IsTargetDevice = false;
815	OMPBuilder.initialize();
816	F->setName("func");
817	IRBuilder<> Builder(BB);
818
819	BasicBlock *EnterBB = BasicBlock::Create(Context&: Ctx, Name: "parallel.enter", Parent: F);
820	Builder.CreateBr(Dest: EnterBB);
821	Builder.SetInsertPoint(EnterBB);
822	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
823
824	unsigned NumInnerBodiesGenerated = 0;
825	unsigned NumOuterBodiesGenerated = 0;
826	unsigned NumFinalizationPoints = 0;
827
828	auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
829	++NumInnerBodiesGenerated;
830	};
831
832	auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
833	Value &Orig, Value &Inner,
834	Value *&ReplacementValue) -> InsertPointTy {
835	// Trivial copy (=firstprivate).
836	Builder.restoreIP(IP: AllocaIP);
837	Type *VTy = ReplacementValue->getType();
838	Value *V = Builder.CreateLoad(Ty: VTy, Ptr: &Inner, Name: Orig.getName() + ".reload");
839	ReplacementValue = Builder.CreateAlloca(Ty: VTy, ArraySize: 0, Name: Orig.getName() + ".copy");
840	Builder.restoreIP(IP: CodeGenIP);
841	Builder.CreateStore(Val: V, Ptr: ReplacementValue);
842	return CodeGenIP;
843	};
844
845	auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
846
847	auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
848	++NumOuterBodiesGenerated;
849	Builder.restoreIP(IP: CodeGenIP);
850	BasicBlock *CGBB = CodeGenIP.getBlock();
851	BasicBlock *NewBB = SplitBlock(Old: CGBB, SplitPt: &*CodeGenIP.getPoint());
852	CGBB->getTerminator()->eraseFromParent();
853	;
854
855	IRBuilder<>::InsertPoint AfterIP = OMPBuilder.createParallel(
856	InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB,
857	FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
858
859	Builder.restoreIP(IP: AfterIP);
860	Builder.CreateBr(Dest: NewBB);
861	};
862
863	IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
864	F->getEntryBlock().getFirstInsertionPt());
865	IRBuilder<>::InsertPoint AfterIP =
866	OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB,
867	nullptr, nullptr, OMP_PROC_BIND_default, false);
868
869	EXPECT_EQ(NumInnerBodiesGenerated, 1U);
870	EXPECT_EQ(NumOuterBodiesGenerated, 1U);
871	EXPECT_EQ(NumFinalizationPoints, 2U);
872
873	Builder.restoreIP(IP: AfterIP);
874	Builder.CreateRetVoid();
875
876	OMPBuilder.finalize();
877
878	EXPECT_EQ(M->size(), 5U);
879	for (Function &OutlinedFn : *M) {
880	if (F == &OutlinedFn || OutlinedFn.isDeclaration())
881	continue;
882	EXPECT_FALSE(verifyModule(*M, &errs()));
883	EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind));
884	EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias));
885	EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias));
886
887	EXPECT_TRUE(OutlinedFn.hasInternalLinkage());
888	EXPECT_EQ(OutlinedFn.arg_size(), 2U);
889
890	EXPECT_EQ(OutlinedFn.getNumUses(), 1U);
891	User *Usr = OutlinedFn.user_back();
892	ASSERT_TRUE(isa<CallInst>(Usr));
893	CallInst *ForkCI = dyn_cast<CallInst>(Val: Usr);
894	ASSERT_NE(ForkCI, nullptr);
895
896	EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
897	EXPECT_EQ(ForkCI->arg_size(), 3U);
898	EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
899	EXPECT_EQ(ForkCI->getArgOperand(1),
900	ConstantInt::get(Type::getInt32Ty(Ctx), 0U));
901	EXPECT_EQ(ForkCI, Usr);
902	}
903	}
904
905	TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
906	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
907	OpenMPIRBuilder OMPBuilder(*M);
908	OMPBuilder.Config.IsTargetDevice = false;
909	OMPBuilder.initialize();
910	F->setName("func");
911	IRBuilder<> Builder(BB);
912
913	BasicBlock *EnterBB = BasicBlock::Create(Context&: Ctx, Name: "parallel.enter", Parent: F);
914	Builder.CreateBr(Dest: EnterBB);
915	Builder.SetInsertPoint(EnterBB);
916	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
917
918	unsigned NumInnerBodiesGenerated = 0;
919	unsigned NumOuterBodiesGenerated = 0;
920	unsigned NumFinalizationPoints = 0;
921
922	auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
923	++NumInnerBodiesGenerated;
924	};
925
926	auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
927	Value &Orig, Value &Inner,
928	Value *&ReplacementValue) -> InsertPointTy {
929	// Trivial copy (=firstprivate).
930	Builder.restoreIP(IP: AllocaIP);
931	Type *VTy = ReplacementValue->getType();
932	Value *V = Builder.CreateLoad(Ty: VTy, Ptr: &Inner, Name: Orig.getName() + ".reload");
933	ReplacementValue = Builder.CreateAlloca(Ty: VTy, ArraySize: 0, Name: Orig.getName() + ".copy");
934	Builder.restoreIP(IP: CodeGenIP);
935	Builder.CreateStore(Val: V, Ptr: ReplacementValue);
936	return CodeGenIP;
937	};
938
939	auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
940
941	auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
942	++NumOuterBodiesGenerated;
943	Builder.restoreIP(IP: CodeGenIP);
944	BasicBlock *CGBB = CodeGenIP.getBlock();
945	BasicBlock *NewBB1 = SplitBlock(Old: CGBB, SplitPt: &*CodeGenIP.getPoint());
946	BasicBlock *NewBB2 = SplitBlock(Old: NewBB1, SplitPt: &*NewBB1->getFirstInsertionPt());
947	CGBB->getTerminator()->eraseFromParent();
948	;
949	NewBB1->getTerminator()->eraseFromParent();
950	;
951
952	IRBuilder<>::InsertPoint AfterIP1 = OMPBuilder.createParallel(
953	InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB,
954	FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
955
956	Builder.restoreIP(IP: AfterIP1);
957	Builder.CreateBr(Dest: NewBB1);
958
959	IRBuilder<>::InsertPoint AfterIP2 = OMPBuilder.createParallel(
960	InsertPointTy(NewBB1, NewBB1->end()), AllocaIP, InnerBodyGenCB, PrivCB,
961	FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
962
963	Builder.restoreIP(IP: AfterIP2);
964	Builder.CreateBr(Dest: NewBB2);
965	};
966
967	IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
968	F->getEntryBlock().getFirstInsertionPt());
969	IRBuilder<>::InsertPoint AfterIP =
970	OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB,
971	nullptr, nullptr, OMP_PROC_BIND_default, false);
972
973	EXPECT_EQ(NumInnerBodiesGenerated, 2U);
974	EXPECT_EQ(NumOuterBodiesGenerated, 1U);
975	EXPECT_EQ(NumFinalizationPoints, 3U);
976
977	Builder.restoreIP(IP: AfterIP);
978	Builder.CreateRetVoid();
979
980	OMPBuilder.finalize();
981
982	EXPECT_EQ(M->size(), 6U);
983	for (Function &OutlinedFn : *M) {
984	if (F == &OutlinedFn || OutlinedFn.isDeclaration())
985	continue;
986	EXPECT_FALSE(verifyModule(*M, &errs()));
987	EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind));
988	EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias));
989	EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias));
990
991	EXPECT_TRUE(OutlinedFn.hasInternalLinkage());
992	EXPECT_EQ(OutlinedFn.arg_size(), 2U);
993
994	unsigned NumAllocas = 0;
995	for (Instruction &I : instructions(F&: OutlinedFn))
996	NumAllocas += isa<AllocaInst>(Val: I);
997	EXPECT_EQ(NumAllocas, 1U);
998
999	EXPECT_EQ(OutlinedFn.getNumUses(), 1U);
1000	User *Usr = OutlinedFn.user_back();
1001	ASSERT_TRUE(isa<CallInst>(Usr));
1002	CallInst *ForkCI = dyn_cast<CallInst>(Val: Usr);
1003	ASSERT_NE(ForkCI, nullptr);
1004
1005	EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
1006	EXPECT_EQ(ForkCI->arg_size(), 3U);
1007	EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
1008	EXPECT_EQ(ForkCI->getArgOperand(1),
1009	ConstantInt::get(Type::getInt32Ty(Ctx), 0U));
1010	EXPECT_EQ(ForkCI, Usr);
1011	}
1012	}
1013
1014	TEST_F(OpenMPIRBuilderTest, ParallelIfCond) {
1015	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1016	OpenMPIRBuilder OMPBuilder(*M);
1017	OMPBuilder.Config.IsTargetDevice = false;
1018	OMPBuilder.initialize();
1019	F->setName("func");
1020	IRBuilder<> Builder(BB);
1021
1022	BasicBlock *EnterBB = BasicBlock::Create(Context&: Ctx, Name: "parallel.enter", Parent: F);
1023	Builder.CreateBr(Dest: EnterBB);
1024	Builder.SetInsertPoint(EnterBB);
1025	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1026
1027	AllocaInst *PrivAI = nullptr;
1028
1029	unsigned NumBodiesGenerated = 0;
1030	unsigned NumPrivatizedVars = 0;
1031	unsigned NumFinalizationPoints = 0;
1032
1033	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
1034	++NumBodiesGenerated;
1035
1036	Builder.restoreIP(IP: AllocaIP);
1037	PrivAI = Builder.CreateAlloca(Ty: F->arg_begin()->getType());
1038	Builder.CreateStore(Val: F->arg_begin(), Ptr: PrivAI);
1039
1040	Builder.restoreIP(IP: CodeGenIP);
1041	Value *PrivLoad =
1042	Builder.CreateLoad(Ty: PrivAI->getAllocatedType(), Ptr: PrivAI, Name: "local.use");
1043	Value *Cmp = Builder.CreateICmpNE(LHS: F->arg_begin(), RHS: PrivLoad);
1044	Instruction *ThenTerm, *ElseTerm;
1045	SplitBlockAndInsertIfThenElse(Cond: Cmp, SplitBefore: &*Builder.GetInsertPoint(), ThenTerm: &ThenTerm,
1046	ElseTerm: &ElseTerm);
1047	};
1048
1049	auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1050	Value &Orig, Value &Inner,
1051	Value *&ReplacementValue) -> InsertPointTy {
1052	++NumPrivatizedVars;
1053
1054	if (!isa<AllocaInst>(Val: Orig)) {
1055	EXPECT_EQ(&Orig, F->arg_begin());
1056	ReplacementValue = &Inner;
1057	return CodeGenIP;
1058	}
1059
1060	// Since the original value is an allocation, it has a pointer type and
1061	// therefore no additional wrapping should happen.
1062	EXPECT_EQ(&Orig, &Inner);
1063
1064	// Trivial copy (=firstprivate).
1065	Builder.restoreIP(IP: AllocaIP);
1066	Type *VTy = ReplacementValue->getType();
1067	Value *V = Builder.CreateLoad(Ty: VTy, Ptr: &Inner, Name: Orig.getName() + ".reload");
1068	ReplacementValue = Builder.CreateAlloca(Ty: VTy, ArraySize: 0, Name: Orig.getName() + ".copy");
1069	Builder.restoreIP(IP: CodeGenIP);
1070	Builder.CreateStore(Val: V, Ptr: ReplacementValue);
1071	return CodeGenIP;
1072	};
1073
1074	auto FiniCB = [&](InsertPointTy CodeGenIP) {
1075	++NumFinalizationPoints;
1076	// No destructors.
1077	};
1078
1079	IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
1080	F->getEntryBlock().getFirstInsertionPt());
1081	IRBuilder<>::InsertPoint AfterIP =
1082	OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1083	Builder.CreateIsNotNull(F->arg_begin()),
1084	nullptr, OMP_PROC_BIND_default, false);
1085
1086	EXPECT_EQ(NumBodiesGenerated, 1U);
1087	EXPECT_EQ(NumPrivatizedVars, 1U);
1088	EXPECT_EQ(NumFinalizationPoints, 1U);
1089
1090	Builder.restoreIP(IP: AfterIP);
1091	Builder.CreateRetVoid();
1092	OMPBuilder.finalize();
1093
1094	EXPECT_NE(PrivAI, nullptr);
1095	Function *OutlinedFn = PrivAI->getFunction();
1096	EXPECT_NE(F, OutlinedFn);
1097	EXPECT_FALSE(verifyModule(*M, &errs()));
1098
1099	EXPECT_TRUE(OutlinedFn->hasInternalLinkage());
1100	EXPECT_EQ(OutlinedFn->arg_size(), 3U);
1101
1102	EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());
1103	ASSERT_EQ(OutlinedFn->getNumUses(), 1U);
1104
1105	CallInst *ForkCI = nullptr;
1106	for (User *Usr : OutlinedFn->users()) {
1107	ASSERT_TRUE(isa<CallInst>(Usr));
1108	ForkCI = cast<CallInst>(Val: Usr);
1109	}
1110
1111	EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call_if");
1112	EXPECT_EQ(ForkCI->arg_size(), 5U);
1113	EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
1114	EXPECT_EQ(ForkCI->getArgOperand(1),
1115	ConstantInt::get(Type::getInt32Ty(Ctx), 1));
1116	EXPECT_EQ(ForkCI->getArgOperand(3)->getType(), Type::getInt32Ty(Ctx));
1117	}
1118
1119	TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
1120	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1121	OpenMPIRBuilder OMPBuilder(*M);
1122	OMPBuilder.Config.IsTargetDevice = false;
1123	OMPBuilder.initialize();
1124	F->setName("func");
1125	IRBuilder<> Builder(BB);
1126
1127	BasicBlock *EnterBB = BasicBlock::Create(Context&: Ctx, Name: "parallel.enter", Parent: F);
1128	Builder.CreateBr(Dest: EnterBB);
1129	Builder.SetInsertPoint(EnterBB);
1130	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1131
1132	unsigned NumBodiesGenerated = 0;
1133	unsigned NumPrivatizedVars = 0;
1134	unsigned NumFinalizationPoints = 0;
1135
1136	CallInst *CheckedBarrier = nullptr;
1137	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
1138	++NumBodiesGenerated;
1139
1140	Builder.restoreIP(IP: CodeGenIP);
1141
1142	// Create three barriers, two cancel barriers but only one checked.
1143	Function *CBFn, *BFn;
1144
1145	Builder.restoreIP(
1146	OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel));
1147
1148	CBFn = M->getFunction(Name: "__kmpc_cancel_barrier");
1149	BFn = M->getFunction(Name: "__kmpc_barrier");
1150	ASSERT_NE(CBFn, nullptr);
1151	ASSERT_EQ(BFn, nullptr);
1152	ASSERT_EQ(CBFn->getNumUses(), 1U);
1153	ASSERT_TRUE(isa<CallInst>(CBFn->user_back()));
1154	ASSERT_EQ(CBFn->user_back()->getNumUses(), 1U);
1155	CheckedBarrier = cast<CallInst>(Val: CBFn->user_back());
1156
1157	Builder.restoreIP(
1158	OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel, true));
1159	CBFn = M->getFunction(Name: "__kmpc_cancel_barrier");
1160	BFn = M->getFunction(Name: "__kmpc_barrier");
1161	ASSERT_NE(CBFn, nullptr);
1162	ASSERT_NE(BFn, nullptr);
1163	ASSERT_EQ(CBFn->getNumUses(), 1U);
1164	ASSERT_EQ(BFn->getNumUses(), 1U);
1165	ASSERT_TRUE(isa<CallInst>(BFn->user_back()));
1166	ASSERT_EQ(BFn->user_back()->getNumUses(), 0U);
1167
1168	Builder.restoreIP(OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel,
1169	false, false));
1170	ASSERT_EQ(CBFn->getNumUses(), 2U);
1171	ASSERT_EQ(BFn->getNumUses(), 1U);
1172	ASSERT_TRUE(CBFn->user_back() != CheckedBarrier);
1173	ASSERT_TRUE(isa<CallInst>(CBFn->user_back()));
1174	ASSERT_EQ(CBFn->user_back()->getNumUses(), 0U);
1175	};
1176
1177	auto PrivCB = [&](InsertPointTy, InsertPointTy, Value &V, Value &,
1178	Value *&) -> InsertPointTy {
1179	++NumPrivatizedVars;
1180	llvm_unreachable("No privatization callback call expected!");
1181	};
1182
1183	FunctionType *FakeDestructorTy =
1184	FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: {Type::getInt32Ty(C&: Ctx)},
1185	/*isVarArg=*/false);
1186	auto *FakeDestructor = Function::Create(
1187	Ty: FakeDestructorTy, Linkage: Function::ExternalLinkage, N: "fakeDestructor", M: M.get());
1188
1189	auto FiniCB = [&](InsertPointTy IP) {
1190	++NumFinalizationPoints;
1191	Builder.restoreIP(IP);
1192	Builder.CreateCall(Callee: FakeDestructor,
1193	Args: {Builder.getInt32(C: NumFinalizationPoints)});
1194	};
1195
1196	IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
1197	F->getEntryBlock().getFirstInsertionPt());
1198	IRBuilder<>::InsertPoint AfterIP =
1199	OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1200	Builder.CreateIsNotNull(F->arg_begin()),
1201	nullptr, OMP_PROC_BIND_default, true);
1202
1203	EXPECT_EQ(NumBodiesGenerated, 1U);
1204	EXPECT_EQ(NumPrivatizedVars, 0U);
1205	EXPECT_EQ(NumFinalizationPoints, 2U);
1206	EXPECT_EQ(FakeDestructor->getNumUses(), 2U);
1207
1208	Builder.restoreIP(IP: AfterIP);
1209	Builder.CreateRetVoid();
1210	OMPBuilder.finalize();
1211
1212	EXPECT_FALSE(verifyModule(*M, &errs()));
1213
1214	BasicBlock *ExitBB = nullptr;
1215	for (const User *Usr : FakeDestructor->users()) {
1216	const CallInst *CI = dyn_cast<CallInst>(Val: Usr);
1217	ASSERT_EQ(CI->getCalledFunction(), FakeDestructor);
1218	ASSERT_TRUE(isa<BranchInst>(CI->getNextNode()));
1219	ASSERT_EQ(CI->getNextNode()->getNumSuccessors(), 1U);
1220	if (ExitBB)
1221	ASSERT_EQ(CI->getNextNode()->getSuccessor(0), ExitBB);
1222	else
1223	ExitBB = CI->getNextNode()->getSuccessor(Idx: 0);
1224	ASSERT_EQ(ExitBB->size(), 1U);
1225	if (!isa<ReturnInst>(Val: ExitBB->front())) {
1226	ASSERT_TRUE(isa<BranchInst>(ExitBB->front()));
1227	ASSERT_EQ(cast<BranchInst>(ExitBB->front()).getNumSuccessors(), 1U);
1228	ASSERT_TRUE(isa<ReturnInst>(
1229	cast<BranchInst>(ExitBB->front()).getSuccessor(0)->front()));
1230	}
1231	}
1232	}
1233
1234	TEST_F(OpenMPIRBuilderTest, ParallelForwardAsPointers) {
1235	OpenMPIRBuilder OMPBuilder(*M);
1236	OMPBuilder.Config.IsTargetDevice = false;
1237	OMPBuilder.initialize();
1238	F->setName("func");
1239	IRBuilder<> Builder(BB);
1240	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1241	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1242
1243	Type *I32Ty = Type::getInt32Ty(C&: M->getContext());
1244	Type *PtrTy = PointerType::get(C&: M->getContext(), AddressSpace: 0);
1245	Type *StructTy = StructType::get(elt1: I32Ty, elts: PtrTy);
1246	Type *VoidTy = Type::getVoidTy(C&: M->getContext());
1247	FunctionCallee RetI32Func = M->getOrInsertFunction(Name: "ret_i32", RetTy: I32Ty);
1248	FunctionCallee TakeI32Func =
1249	M->getOrInsertFunction(Name: "take_i32", RetTy: VoidTy, Args: I32Ty);
1250	FunctionCallee RetI32PtrFunc = M->getOrInsertFunction(Name: "ret_i32ptr", RetTy: PtrTy);
1251	FunctionCallee TakeI32PtrFunc =
1252	M->getOrInsertFunction(Name: "take_i32ptr", RetTy: VoidTy, Args: PtrTy);
1253	FunctionCallee RetStructFunc = M->getOrInsertFunction(Name: "ret_struct", RetTy: StructTy);
1254	FunctionCallee TakeStructFunc =
1255	M->getOrInsertFunction(Name: "take_struct", RetTy: VoidTy, Args: StructTy);
1256	FunctionCallee RetStructPtrFunc =
1257	M->getOrInsertFunction(Name: "ret_structptr", RetTy: PtrTy);
1258	FunctionCallee TakeStructPtrFunc =
1259	M->getOrInsertFunction(Name: "take_structPtr", RetTy: VoidTy, Args: PtrTy);
1260	Value *I32Val = Builder.CreateCall(Callee: RetI32Func);
1261	Value *I32PtrVal = Builder.CreateCall(Callee: RetI32PtrFunc);
1262	Value *StructVal = Builder.CreateCall(Callee: RetStructFunc);
1263	Value *StructPtrVal = Builder.CreateCall(Callee: RetStructPtrFunc);
1264
1265	Instruction *Internal;
1266	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
1267	IRBuilder<>::InsertPointGuard Guard(Builder);
1268	Builder.restoreIP(IP: CodeGenIP);
1269	Internal = Builder.CreateCall(Callee: TakeI32Func, Args: I32Val);
1270	Builder.CreateCall(Callee: TakeI32PtrFunc, Args: I32PtrVal);
1271	Builder.CreateCall(Callee: TakeStructFunc, Args: StructVal);
1272	Builder.CreateCall(Callee: TakeStructPtrFunc, Args: StructPtrVal);
1273	};
1274	auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &,
1275	Value &Inner, Value *&ReplacementValue) {
1276	ReplacementValue = &Inner;
1277	return CodeGenIP;
1278	};
1279	auto FiniCB = [](InsertPointTy) {};
1280
1281	IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
1282	F->getEntryBlock().getFirstInsertionPt());
1283	IRBuilder<>::InsertPoint AfterIP =
1284	OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1285	nullptr, nullptr, OMP_PROC_BIND_default, false);
1286	Builder.restoreIP(IP: AfterIP);
1287	Builder.CreateRetVoid();
1288
1289	OMPBuilder.finalize();
1290
1291	EXPECT_FALSE(verifyModule(*M, &errs()));
1292	Function *OutlinedFn = Internal->getFunction();
1293
1294	Type *Arg2Type = OutlinedFn->getArg(i: 2)->getType();
1295	EXPECT_TRUE(Arg2Type->isPointerTy());
1296	}
1297
1298	TEST_F(OpenMPIRBuilderTest, CanonicalLoopSimple) {
1299	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1300	OpenMPIRBuilder OMPBuilder(*M);
1301	OMPBuilder.initialize();
1302	IRBuilder<> Builder(BB);
1303	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1304	Value *TripCount = F->getArg(i: 0);
1305
1306	unsigned NumBodiesGenerated = 0;
1307	auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {
1308	NumBodiesGenerated += 1;
1309
1310	Builder.restoreIP(IP: CodeGenIP);
1311
1312	Value *Cmp = Builder.CreateICmpEQ(LHS: LC, RHS: TripCount);
1313	Instruction *ThenTerm, *ElseTerm;
1314	SplitBlockAndInsertIfThenElse(Cond: Cmp, SplitBefore: CodeGenIP.getBlock()->getTerminator(),
1315	ThenTerm: &ThenTerm, ElseTerm: &ElseTerm);
1316	};
1317
1318	CanonicalLoopInfo *Loop =
1319	OMPBuilder.createCanonicalLoop(Loc, BodyGenCB: LoopBodyGenCB, TripCount);
1320
1321	Builder.restoreIP(IP: Loop->getAfterIP());
1322	ReturnInst *RetInst = Builder.CreateRetVoid();
1323	OMPBuilder.finalize();
1324
1325	Loop->assertOK();
1326	EXPECT_FALSE(verifyModule(*M, &errs()));
1327
1328	EXPECT_EQ(NumBodiesGenerated, 1U);
1329
1330	// Verify control flow structure (in addition to Loop->assertOK()).
1331	EXPECT_EQ(Loop->getPreheader()->getSinglePredecessor(), &F->getEntryBlock());
1332	EXPECT_EQ(Loop->getAfter(), Builder.GetInsertBlock());
1333
1334	Instruction *IndVar = Loop->getIndVar();
1335	EXPECT_TRUE(isa<PHINode>(IndVar));
1336	EXPECT_EQ(IndVar->getType(), TripCount->getType());
1337	EXPECT_EQ(IndVar->getParent(), Loop->getHeader());
1338
1339	EXPECT_EQ(Loop->getTripCount(), TripCount);
1340
1341	BasicBlock *Body = Loop->getBody();
1342	Instruction *CmpInst = &Body->front();
1343	EXPECT_TRUE(isa<ICmpInst>(CmpInst));
1344	EXPECT_EQ(CmpInst->getOperand(0), IndVar);
1345
1346	BasicBlock *LatchPred = Loop->getLatch()->getSinglePredecessor();
1347	EXPECT_TRUE(llvm::all_of(successors(Body), [=](BasicBlock *SuccBB) {
1348	return SuccBB->getSingleSuccessor() == LatchPred;
1349	}));
1350
1351	EXPECT_EQ(&Loop->getAfter()->front(), RetInst);
1352	}
1353
1354	TEST_F(OpenMPIRBuilderTest, CanonicalLoopBounds) {
1355	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1356	OpenMPIRBuilder OMPBuilder(*M);
1357	OMPBuilder.initialize();
1358	IRBuilder<> Builder(BB);
1359
1360	// Check the trip count is computed correctly. We generate the canonical loop
1361	// but rely on the IRBuilder's constant folder to compute the final result
1362	// since all inputs are constant. To verify overflow situations, limit the
1363	// trip count / loop counter widths to 16 bits.
1364	auto EvalTripCount = [&](int64_t Start, int64_t Stop, int64_t Step,
1365	bool IsSigned, bool InclusiveStop) -> int64_t {
1366	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1367	Type *LCTy = Type::getInt16Ty(C&: Ctx);
1368	Value *StartVal = ConstantInt::get(Ty: LCTy, V: Start);
1369	Value *StopVal = ConstantInt::get(Ty: LCTy, V: Stop);
1370	Value *StepVal = ConstantInt::get(Ty: LCTy, V: Step);
1371	auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {};
1372	CanonicalLoopInfo *Loop =
1373	OMPBuilder.createCanonicalLoop(Loc, BodyGenCB: LoopBodyGenCB, Start: StartVal, Stop: StopVal,
1374	Step: StepVal, IsSigned, InclusiveStop);
1375	Loop->assertOK();
1376	Builder.restoreIP(IP: Loop->getAfterIP());
1377	Value *TripCount = Loop->getTripCount();
1378	return cast<ConstantInt>(Val: TripCount)->getValue().getZExtValue();
1379	};
1380
1381	EXPECT_EQ(EvalTripCount(0, 0, 1, false, false), 0);
1382	EXPECT_EQ(EvalTripCount(0, 1, 2, false, false), 1);
1383	EXPECT_EQ(EvalTripCount(0, 42, 1, false, false), 42);
1384	EXPECT_EQ(EvalTripCount(0, 42, 2, false, false), 21);
1385	EXPECT_EQ(EvalTripCount(21, 42, 1, false, false), 21);
1386	EXPECT_EQ(EvalTripCount(0, 5, 5, false, false), 1);
1387	EXPECT_EQ(EvalTripCount(0, 9, 5, false, false), 2);
1388	EXPECT_EQ(EvalTripCount(0, 11, 5, false, false), 3);
1389	EXPECT_EQ(EvalTripCount(0, 0xFFFF, 1, false, false), 0xFFFF);
1390	EXPECT_EQ(EvalTripCount(0xFFFF, 0, 1, false, false), 0);
1391	EXPECT_EQ(EvalTripCount(0xFFFE, 0xFFFF, 1, false, false), 1);
1392	EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0x100, false, false), 0x100);
1393	EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFF, false, false), 1);
1394
1395	EXPECT_EQ(EvalTripCount(0, 6, 5, false, false), 2);
1396	EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFE, false, false), 2);
1397	EXPECT_EQ(EvalTripCount(0, 0, 1, false, true), 1);
1398	EXPECT_EQ(EvalTripCount(0, 0, 0xFFFF, false, true), 1);
1399	EXPECT_EQ(EvalTripCount(0, 0xFFFE, 1, false, true), 0xFFFF);
1400	EXPECT_EQ(EvalTripCount(0, 0xFFFE, 2, false, true), 0x8000);
1401
1402	EXPECT_EQ(EvalTripCount(0, 0, -1, true, false), 0);
1403	EXPECT_EQ(EvalTripCount(0, 1, -1, true, true), 0);
1404	EXPECT_EQ(EvalTripCount(20, 5, -5, true, false), 3);
1405	EXPECT_EQ(EvalTripCount(20, 5, -5, true, true), 4);
1406	EXPECT_EQ(EvalTripCount(-4, -2, 2, true, false), 1);
1407	EXPECT_EQ(EvalTripCount(-4, -3, 2, true, false), 1);
1408	EXPECT_EQ(EvalTripCount(-4, -2, 2, true, true), 2);
1409
1410	EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, false), 0x8000);
1411	EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, true), 0x8001);
1412	EXPECT_EQ(EvalTripCount(INT16_MIN, 0x7FFF, 1, true, false), 0xFFFF);
1413	EXPECT_EQ(EvalTripCount(INT16_MIN + 1, 0x7FFF, 1, true, true), 0xFFFF);
1414	EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 0x7FFF, true, false), 2);
1415	EXPECT_EQ(EvalTripCount(0x7FFF, 0, -1, true, false), 0x7FFF);
1416	EXPECT_EQ(EvalTripCount(0, INT16_MIN, -1, true, false), 0x8000);
1417	EXPECT_EQ(EvalTripCount(0, INT16_MIN, -16, true, false), 0x800);
1418	EXPECT_EQ(EvalTripCount(0x7FFF, INT16_MIN, -1, true, false), 0xFFFF);
1419	EXPECT_EQ(EvalTripCount(0x7FFF, 1, INT16_MIN, true, false), 1);
1420	EXPECT_EQ(EvalTripCount(0x7FFF, -1, INT16_MIN, true, true), 2);
1421
1422	// Finalize the function and verify it.
1423	Builder.CreateRetVoid();
1424	OMPBuilder.finalize();
1425	EXPECT_FALSE(verifyModule(*M, &errs()));
1426	}
1427
1428	TEST_F(OpenMPIRBuilderTest, CollapseNestedLoops) {
1429	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1430	OpenMPIRBuilder OMPBuilder(*M);
1431	OMPBuilder.initialize();
1432	F->setName("func");
1433
1434	IRBuilder<> Builder(BB);
1435
1436	Type *LCTy = F->getArg(i: 0)->getType();
1437	Constant *One = ConstantInt::get(Ty: LCTy, V: 1);
1438	Constant *Two = ConstantInt::get(Ty: LCTy, V: 2);
1439	Value *OuterTripCount =
1440	Builder.CreateAdd(LHS: F->getArg(i: 0), RHS: Two, Name: "tripcount.outer");
1441	Value *InnerTripCount =
1442	Builder.CreateAdd(LHS: F->getArg(i: 0), RHS: One, Name: "tripcount.inner");
1443
1444	// Fix an insertion point for ComputeIP.
1445	BasicBlock *LoopNextEnter =
1446	BasicBlock::Create(Context&: M->getContext(), Name: "loopnest.enter", Parent: F,
1447	InsertBefore: Builder.GetInsertBlock()->getNextNode());
1448	BranchInst *EnterBr = Builder.CreateBr(Dest: LoopNextEnter);
1449	InsertPointTy ComputeIP{EnterBr->getParent(), EnterBr->getIterator()};
1450
1451	Builder.SetInsertPoint(LoopNextEnter);
1452	OpenMPIRBuilder::LocationDescription OuterLoc(Builder.saveIP(), DL);
1453
1454	CanonicalLoopInfo *InnerLoop = nullptr;
1455	CallInst *InbetweenLead = nullptr;
1456	CallInst *InbetweenTrail = nullptr;
1457	CallInst *Call = nullptr;
1458	auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP, Value *OuterLC) {
1459	Builder.restoreIP(IP: OuterCodeGenIP);
1460	InbetweenLead =
1461	createPrintfCall(Builder, FormatStr: "In-between lead i=%d\\n", Values: {OuterLC});
1462
1463	auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,
1464	Value *InnerLC) {
1465	Builder.restoreIP(IP: InnerCodeGenIP);
1466	Call = createPrintfCall(Builder, FormatStr: "body i=%d j=%d\\n", Values: {OuterLC, InnerLC});
1467	};
1468	InnerLoop = OMPBuilder.createCanonicalLoop(
1469	Loc: Builder.saveIP(), BodyGenCB: InnerLoopBodyGenCB, TripCount: InnerTripCount, Name: "inner");
1470
1471	Builder.restoreIP(IP: InnerLoop->getAfterIP());
1472	InbetweenTrail =
1473	createPrintfCall(Builder, FormatStr: "In-between trail i=%d\\n", Values: {OuterLC});
1474	};
1475	CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
1476	Loc: OuterLoc, BodyGenCB: OuterLoopBodyGenCB, TripCount: OuterTripCount, Name: "outer");
1477
1478	// Finish the function.
1479	Builder.restoreIP(IP: OuterLoop->getAfterIP());
1480	Builder.CreateRetVoid();
1481
1482	CanonicalLoopInfo *Collapsed =
1483	OMPBuilder.collapseLoops(DL, Loops: {OuterLoop, InnerLoop}, ComputeIP);
1484
1485	OMPBuilder.finalize();
1486	EXPECT_FALSE(verifyModule(*M, &errs()));
1487
1488	// Verify control flow and BB order.
1489	BasicBlock *RefOrder[] = {
1490	Collapsed->getPreheader(), Collapsed->getHeader(),
1491	Collapsed->getCond(), Collapsed->getBody(),
1492	InbetweenLead->getParent(), Call->getParent(),
1493	InbetweenTrail->getParent(), Collapsed->getLatch(),
1494	Collapsed->getExit(), Collapsed->getAfter(),
1495	};
1496	EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1497	EXPECT_TRUE(verifyListOrder(F, RefOrder));
1498
1499	// Verify the total trip count.
1500	auto *TripCount = cast<MulOperator>(Val: Collapsed->getTripCount());
1501	EXPECT_EQ(TripCount->getOperand(0), OuterTripCount);
1502	EXPECT_EQ(TripCount->getOperand(1), InnerTripCount);
1503
1504	// Verify the changed indvar.
1505	auto *OuterIV = cast<BinaryOperator>(Val: Call->getOperand(i_nocapture: 1));
1506	EXPECT_EQ(OuterIV->getOpcode(), Instruction::UDiv);
1507	EXPECT_EQ(OuterIV->getParent(), Collapsed->getBody());
1508	EXPECT_EQ(OuterIV->getOperand(1), InnerTripCount);
1509	EXPECT_EQ(OuterIV->getOperand(0), Collapsed->getIndVar());
1510
1511	auto *InnerIV = cast<BinaryOperator>(Val: Call->getOperand(i_nocapture: 2));
1512	EXPECT_EQ(InnerIV->getOpcode(), Instruction::URem);
1513	EXPECT_EQ(InnerIV->getParent(), Collapsed->getBody());
1514	EXPECT_EQ(InnerIV->getOperand(0), Collapsed->getIndVar());
1515	EXPECT_EQ(InnerIV->getOperand(1), InnerTripCount);
1516
1517	EXPECT_EQ(InbetweenLead->getOperand(1), OuterIV);
1518	EXPECT_EQ(InbetweenTrail->getOperand(1), OuterIV);
1519	}
1520
1521	TEST_F(OpenMPIRBuilderTest, TileSingleLoop) {
1522	OpenMPIRBuilder OMPBuilder(*M);
1523	CallInst *Call;
1524	BasicBlock *BodyCode;
1525	CanonicalLoopInfo *Loop =
1526	buildSingleLoopFunction(DL, OMPBuilder, UseIVBits: 32, Call: &Call, BodyCode: &BodyCode);
1527
1528	Instruction *OrigIndVar = Loop->getIndVar();
1529	EXPECT_EQ(Call->getOperand(1), OrigIndVar);
1530
1531	// Tile the loop.
1532	Constant *TileSize = ConstantInt::get(Ty: Loop->getIndVarType(), V: APInt(32, 7));
1533	std::vector<CanonicalLoopInfo *> GenLoops =
1534	OMPBuilder.tileLoops(DL, Loops: {Loop}, TileSizes: {TileSize});
1535
1536	OMPBuilder.finalize();
1537	EXPECT_FALSE(verifyModule(*M, &errs()));
1538
1539	EXPECT_EQ(GenLoops.size(), 2u);
1540	CanonicalLoopInfo *Floor = GenLoops[0];
1541	CanonicalLoopInfo *Tile = GenLoops[1];
1542
1543	BasicBlock *RefOrder[] = {
1544	Floor->getPreheader(), Floor->getHeader(), Floor->getCond(),
1545	Floor->getBody(), Tile->getPreheader(), Tile->getHeader(),
1546	Tile->getCond(), Tile->getBody(), BodyCode,
1547	Tile->getLatch(), Tile->getExit(), Tile->getAfter(),
1548	Floor->getLatch(), Floor->getExit(), Floor->getAfter(),
1549	};
1550	EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1551	EXPECT_TRUE(verifyListOrder(F, RefOrder));
1552
1553	// Check the induction variable.
1554	EXPECT_EQ(Call->getParent(), BodyCode);
1555	auto *Shift = cast<AddOperator>(Val: Call->getOperand(i_nocapture: 1));
1556	EXPECT_EQ(cast<Instruction>(Shift)->getParent(), Tile->getBody());
1557	EXPECT_EQ(Shift->getOperand(1), Tile->getIndVar());
1558	auto *Scale = cast<MulOperator>(Val: Shift->getOperand(i_nocapture: 0));
1559	EXPECT_EQ(cast<Instruction>(Scale)->getParent(), Tile->getBody());
1560	EXPECT_EQ(Scale->getOperand(0), TileSize);
1561	EXPECT_EQ(Scale->getOperand(1), Floor->getIndVar());
1562	}
1563
1564	TEST_F(OpenMPIRBuilderTest, TileNestedLoops) {
1565	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1566	OpenMPIRBuilder OMPBuilder(*M);
1567	OMPBuilder.initialize();
1568	F->setName("func");
1569
1570	IRBuilder<> Builder(BB);
1571	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1572	Value *TripCount = F->getArg(i: 0);
1573	Type *LCTy = TripCount->getType();
1574
1575	BasicBlock *BodyCode = nullptr;
1576	CanonicalLoopInfo *InnerLoop = nullptr;
1577	auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP,
1578	llvm::Value *OuterLC) {
1579	auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,
1580	llvm::Value *InnerLC) {
1581	Builder.restoreIP(IP: InnerCodeGenIP);
1582	BodyCode = Builder.GetInsertBlock();
1583
1584	// Add something that consumes the induction variables to the body.
1585	createPrintfCall(Builder, FormatStr: "i=%d j=%d\\n", Values: {OuterLC, InnerLC});
1586	};
1587	InnerLoop = OMPBuilder.createCanonicalLoop(
1588	Loc: OuterCodeGenIP, BodyGenCB: InnerLoopBodyGenCB, TripCount, Name: "inner");
1589	};
1590	CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
1591	Loc, BodyGenCB: OuterLoopBodyGenCB, TripCount, Name: "outer");
1592
1593	// Finalize the function.
1594	Builder.restoreIP(IP: OuterLoop->getAfterIP());
1595	Builder.CreateRetVoid();
1596
1597	// Tile to loop nest.
1598	Constant *OuterTileSize = ConstantInt::get(Ty: LCTy, V: APInt(32, 11));
1599	Constant *InnerTileSize = ConstantInt::get(Ty: LCTy, V: APInt(32, 7));
1600	std::vector<CanonicalLoopInfo *> GenLoops = OMPBuilder.tileLoops(
1601	DL, Loops: {OuterLoop, InnerLoop}, TileSizes: {OuterTileSize, InnerTileSize});
1602
1603	OMPBuilder.finalize();
1604	EXPECT_FALSE(verifyModule(*M, &errs()));
1605
1606	EXPECT_EQ(GenLoops.size(), 4u);
1607	CanonicalLoopInfo *Floor1 = GenLoops[0];
1608	CanonicalLoopInfo *Floor2 = GenLoops[1];
1609	CanonicalLoopInfo *Tile1 = GenLoops[2];
1610	CanonicalLoopInfo *Tile2 = GenLoops[3];
1611
1612	BasicBlock *RefOrder[] = {
1613	Floor1->getPreheader(),
1614	Floor1->getHeader(),
1615	Floor1->getCond(),
1616	Floor1->getBody(),
1617	Floor2->getPreheader(),
1618	Floor2->getHeader(),
1619	Floor2->getCond(),
1620	Floor2->getBody(),
1621	Tile1->getPreheader(),
1622	Tile1->getHeader(),
1623	Tile1->getCond(),
1624	Tile1->getBody(),
1625	Tile2->getPreheader(),
1626	Tile2->getHeader(),
1627	Tile2->getCond(),
1628	Tile2->getBody(),
1629	BodyCode,
1630	Tile2->getLatch(),
1631	Tile2->getExit(),
1632	Tile2->getAfter(),
1633	Tile1->getLatch(),
1634	Tile1->getExit(),
1635	Tile1->getAfter(),
1636	Floor2->getLatch(),
1637	Floor2->getExit(),
1638	Floor2->getAfter(),
1639	Floor1->getLatch(),
1640	Floor1->getExit(),
1641	Floor1->getAfter(),
1642	};
1643	EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1644	EXPECT_TRUE(verifyListOrder(F, RefOrder));
1645	}
1646
1647	TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) {
1648	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1649	OpenMPIRBuilder OMPBuilder(*M);
1650	OMPBuilder.initialize();
1651	F->setName("func");
1652
1653	IRBuilder<> Builder(BB);
1654	Value *TripCount = F->getArg(i: 0);
1655	Type *LCTy = TripCount->getType();
1656
1657	Value *OuterStartVal = ConstantInt::get(Ty: LCTy, V: 2);
1658	Value *OuterStopVal = TripCount;
1659	Value *OuterStep = ConstantInt::get(Ty: LCTy, V: 5);
1660	Value *InnerStartVal = ConstantInt::get(Ty: LCTy, V: 13);
1661	Value *InnerStopVal = TripCount;
1662	Value *InnerStep = ConstantInt::get(Ty: LCTy, V: 3);
1663
1664	// Fix an insertion point for ComputeIP.
1665	BasicBlock *LoopNextEnter =
1666	BasicBlock::Create(Context&: M->getContext(), Name: "loopnest.enter", Parent: F,
1667	InsertBefore: Builder.GetInsertBlock()->getNextNode());
1668	BranchInst *EnterBr = Builder.CreateBr(Dest: LoopNextEnter);
1669	InsertPointTy ComputeIP{EnterBr->getParent(), EnterBr->getIterator()};
1670
1671	InsertPointTy LoopIP{LoopNextEnter, LoopNextEnter->begin()};
1672	OpenMPIRBuilder::LocationDescription Loc({LoopIP, DL});
1673
1674	BasicBlock *BodyCode = nullptr;
1675	CanonicalLoopInfo *InnerLoop = nullptr;
1676	CallInst *Call = nullptr;
1677	auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP,
1678	llvm::Value *OuterLC) {
1679	auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,
1680	llvm::Value *InnerLC) {
1681	Builder.restoreIP(IP: InnerCodeGenIP);
1682	BodyCode = Builder.GetInsertBlock();
1683
1684	// Add something that consumes the induction variable to the body.
1685	Call = createPrintfCall(Builder, FormatStr: "i=%d j=%d\\n", Values: {OuterLC, InnerLC});
1686	};
1687	InnerLoop = OMPBuilder.createCanonicalLoop(
1688	Loc: OuterCodeGenIP, BodyGenCB: InnerLoopBodyGenCB, Start: InnerStartVal, Stop: InnerStopVal,
1689	Step: InnerStep, IsSigned: false, InclusiveStop: false, ComputeIP, Name: "inner");
1690	};
1691	CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
1692	Loc, BodyGenCB: OuterLoopBodyGenCB, Start: OuterStartVal, Stop: OuterStopVal, Step: OuterStep, IsSigned: false,
1693	InclusiveStop: false, ComputeIP, Name: "outer");
1694
1695	// Finalize the function
1696	Builder.restoreIP(IP: OuterLoop->getAfterIP());
1697	Builder.CreateRetVoid();
1698
1699	// Tile the loop nest.
1700	Constant *TileSize0 = ConstantInt::get(Ty: LCTy, V: APInt(32, 11));
1701	Constant *TileSize1 = ConstantInt::get(Ty: LCTy, V: APInt(32, 7));
1702	std::vector<CanonicalLoopInfo *> GenLoops =
1703	OMPBuilder.tileLoops(DL, Loops: {OuterLoop, InnerLoop}, TileSizes: {TileSize0, TileSize1});
1704
1705	OMPBuilder.finalize();
1706	EXPECT_FALSE(verifyModule(*M, &errs()));
1707
1708	EXPECT_EQ(GenLoops.size(), 4u);
1709	CanonicalLoopInfo *Floor0 = GenLoops[0];
1710	CanonicalLoopInfo *Floor1 = GenLoops[1];
1711	CanonicalLoopInfo *Tile0 = GenLoops[2];
1712	CanonicalLoopInfo *Tile1 = GenLoops[3];
1713
1714	BasicBlock *RefOrder[] = {
1715	Floor0->getPreheader(),
1716	Floor0->getHeader(),
1717	Floor0->getCond(),
1718	Floor0->getBody(),
1719	Floor1->getPreheader(),
1720	Floor1->getHeader(),
1721	Floor1->getCond(),
1722	Floor1->getBody(),
1723	Tile0->getPreheader(),
1724	Tile0->getHeader(),
1725	Tile0->getCond(),
1726	Tile0->getBody(),
1727	Tile1->getPreheader(),
1728	Tile1->getHeader(),
1729	Tile1->getCond(),
1730	Tile1->getBody(),
1731	BodyCode,
1732	Tile1->getLatch(),
1733	Tile1->getExit(),
1734	Tile1->getAfter(),
1735	Tile0->getLatch(),
1736	Tile0->getExit(),
1737	Tile0->getAfter(),
1738	Floor1->getLatch(),
1739	Floor1->getExit(),
1740	Floor1->getAfter(),
1741	Floor0->getLatch(),
1742	Floor0->getExit(),
1743	Floor0->getAfter(),
1744	};
1745	EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1746	EXPECT_TRUE(verifyListOrder(F, RefOrder));
1747
1748	EXPECT_EQ(Call->getParent(), BodyCode);
1749
1750	auto *RangeShift0 = cast<AddOperator>(Val: Call->getOperand(i_nocapture: 1));
1751	EXPECT_EQ(RangeShift0->getOperand(1), OuterStartVal);
1752	auto *RangeScale0 = cast<MulOperator>(Val: RangeShift0->getOperand(i_nocapture: 0));
1753	EXPECT_EQ(RangeScale0->getOperand(1), OuterStep);
1754	auto *TileShift0 = cast<AddOperator>(Val: RangeScale0->getOperand(i_nocapture: 0));
1755	EXPECT_EQ(cast<Instruction>(TileShift0)->getParent(), Tile1->getBody());
1756	EXPECT_EQ(TileShift0->getOperand(1), Tile0->getIndVar());
1757	auto *TileScale0 = cast<MulOperator>(Val: TileShift0->getOperand(i_nocapture: 0));
1758	EXPECT_EQ(cast<Instruction>(TileScale0)->getParent(), Tile1->getBody());
1759	EXPECT_EQ(TileScale0->getOperand(0), TileSize0);
1760	EXPECT_EQ(TileScale0->getOperand(1), Floor0->getIndVar());
1761
1762	auto *RangeShift1 = cast<AddOperator>(Val: Call->getOperand(i_nocapture: 2));
1763	EXPECT_EQ(cast<Instruction>(RangeShift1)->getParent(), BodyCode);
1764	EXPECT_EQ(RangeShift1->getOperand(1), InnerStartVal);
1765	auto *RangeScale1 = cast<MulOperator>(Val: RangeShift1->getOperand(i_nocapture: 0));
1766	EXPECT_EQ(cast<Instruction>(RangeScale1)->getParent(), BodyCode);
1767	EXPECT_EQ(RangeScale1->getOperand(1), InnerStep);
1768	auto *TileShift1 = cast<AddOperator>(Val: RangeScale1->getOperand(i_nocapture: 0));
1769	EXPECT_EQ(cast<Instruction>(TileShift1)->getParent(), Tile1->getBody());
1770	EXPECT_EQ(TileShift1->getOperand(1), Tile1->getIndVar());
1771	auto *TileScale1 = cast<MulOperator>(Val: TileShift1->getOperand(i_nocapture: 0));
1772	EXPECT_EQ(cast<Instruction>(TileScale1)->getParent(), Tile1->getBody());
1773	EXPECT_EQ(TileScale1->getOperand(0), TileSize1);
1774	EXPECT_EQ(TileScale1->getOperand(1), Floor1->getIndVar());
1775	}
1776
1777	TEST_F(OpenMPIRBuilderTest, TileSingleLoopCounts) {
1778	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1779	OpenMPIRBuilder OMPBuilder(*M);
1780	OMPBuilder.initialize();
1781	IRBuilder<> Builder(BB);
1782
1783	// Create a loop, tile it, and extract its trip count. All input values are
1784	// constant and IRBuilder evaluates all-constant arithmetic inplace, such that
1785	// the floor trip count itself will be a ConstantInt. Unfortunately we cannot
1786	// do the same for the tile loop.
1787	auto GetFloorCount = [&](int64_t Start, int64_t Stop, int64_t Step,
1788	bool IsSigned, bool InclusiveStop,
1789	int64_t TileSize) -> uint64_t {
1790	OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
1791	Type *LCTy = Type::getInt16Ty(C&: Ctx);
1792	Value *StartVal = ConstantInt::get(Ty: LCTy, V: Start);
1793	Value *StopVal = ConstantInt::get(Ty: LCTy, V: Stop);
1794	Value *StepVal = ConstantInt::get(Ty: LCTy, V: Step);
1795
1796	// Generate a loop.
1797	auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {};
1798	CanonicalLoopInfo *Loop =
1799	OMPBuilder.createCanonicalLoop(Loc, BodyGenCB: LoopBodyGenCB, Start: StartVal, Stop: StopVal,
1800	Step: StepVal, IsSigned, InclusiveStop);
1801	InsertPointTy AfterIP = Loop->getAfterIP();
1802
1803	// Tile the loop.
1804	Value *TileSizeVal = ConstantInt::get(Ty: LCTy, V: TileSize);
1805	std::vector<CanonicalLoopInfo *> GenLoops =
1806	OMPBuilder.tileLoops(DL: Loc.DL, Loops: {Loop}, TileSizes: {TileSizeVal});
1807
1808	// Set the insertion pointer to after loop, where the next loop will be
1809	// emitted.
1810	Builder.restoreIP(IP: AfterIP);
1811
1812	// Extract the trip count.
1813	CanonicalLoopInfo *FloorLoop = GenLoops[0];
1814	Value *FloorTripCount = FloorLoop->getTripCount();
1815	return cast<ConstantInt>(Val: FloorTripCount)->getValue().getZExtValue();
1816	};
1817
1818	// Empty iteration domain.
1819	EXPECT_EQ(GetFloorCount(0, 0, 1, false, false, 7), 0u);
1820	EXPECT_EQ(GetFloorCount(0, -1, 1, false, true, 7), 0u);
1821	EXPECT_EQ(GetFloorCount(-1, -1, -1, true, false, 7), 0u);
1822	EXPECT_EQ(GetFloorCount(-1, 0, -1, true, true, 7), 0u);
1823	EXPECT_EQ(GetFloorCount(-1, -1, 3, true, false, 7), 0u);
1824
1825	// Only complete tiles.
1826	EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u);
1827	EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u);
1828	EXPECT_EQ(GetFloorCount(1, 15, 1, false, false, 7), 2u);
1829	EXPECT_EQ(GetFloorCount(0, -14, -1, true, false, 7), 2u);
1830	EXPECT_EQ(GetFloorCount(-1, -14, -1, true, true, 7), 2u);
1831	EXPECT_EQ(GetFloorCount(0, 3 * 7 * 2, 3, false, false, 7), 2u);
1832
1833	// Only a partial tile.
1834	EXPECT_EQ(GetFloorCount(0, 1, 1, false, false, 7), 1u);
1835	EXPECT_EQ(GetFloorCount(0, 6, 1, false, false, 7), 1u);
1836	EXPECT_EQ(GetFloorCount(-1, 1, 3, true, false, 7), 1u);
1837	EXPECT_EQ(GetFloorCount(-1, -2, -1, true, false, 7), 1u);
1838	EXPECT_EQ(GetFloorCount(0, 2, 3, false, false, 7), 1u);
1839
1840	// Complete and partial tiles.
1841	EXPECT_EQ(GetFloorCount(0, 13, 1, false, false, 7), 2u);
1842	EXPECT_EQ(GetFloorCount(0, 15, 1, false, false, 7), 3u);
1843	EXPECT_EQ(GetFloorCount(-1, -14, -1, true, false, 7), 2u);
1844	EXPECT_EQ(GetFloorCount(0, 3 * 7 * 5 - 1, 3, false, false, 7), 5u);
1845	EXPECT_EQ(GetFloorCount(-1, -3 * 7 * 5, -3, true, false, 7), 5u);
1846
1847	// Close to 16-bit integer range.
1848	EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 1), 0xFFFFu);
1849	EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 7), 0xFFFFu / 7 + 1);
1850	EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, true, 7), 0xFFFFu / 7 + 1);
1851	EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 7), 0xFFFFu / 7 + 1);
1852	EXPECT_EQ(GetFloorCount(-0x7FFF, 0x7FFF, 1, true, true, 7), 0xFFFFu / 7 + 1);
1853	EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, false, 0xFFFF), 1u);
1854	EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 0xFFFF), 1u);
1855
1856	// Finalize the function.
1857	Builder.CreateRetVoid();
1858	OMPBuilder.finalize();
1859
1860	EXPECT_FALSE(verifyModule(*M, &errs()));
1861	}
1862
1863	TEST_F(OpenMPIRBuilderTest, ApplySimd) {
1864	OpenMPIRBuilder OMPBuilder(*M);
1865	MapVector<Value *, Value *> AlignedVars;
1866	CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, UseIVBits: 32);
1867
1868	// Simd-ize the loop.
1869	OMPBuilder.applySimd(CLI, AlignedVars, /* IfCond */ nullptr,
1870	OrderKind::OMP_ORDER_unknown,
1871	/* Simdlen */ nullptr,
1872	/* Safelen */ nullptr);
1873
1874	OMPBuilder.finalize();
1875	EXPECT_FALSE(verifyModule(*M, &errs()));
1876
1877	PassBuilder PB;
1878	FunctionAnalysisManager FAM;
1879	PB.registerFunctionAnalyses(FAM);
1880	LoopInfo &LI = FAM.getResult<LoopAnalysis>(IR&: *F);
1881
1882	const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1883	EXPECT_EQ(TopLvl.size(), 1u);
1884
1885	Loop *L = TopLvl.front();
1886	EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1887	EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1888
1889	// Check for llvm.access.group metadata attached to the printf
1890	// function in the loop body.
1891	BasicBlock *LoopBody = CLI->getBody();
1892	EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
1893	return I.getMetadata("llvm.access.group") != nullptr;
1894	}));
1895	}
1896
1897	TEST_F(OpenMPIRBuilderTest, ApplySimdCustomAligned) {
1898	OpenMPIRBuilder OMPBuilder(*M);
1899	IRBuilder<> Builder(BB);
1900	const int AlignmentValue = 32;
1901	AllocaInst *Alloc1 =
1902	Builder.CreateAlloca(Ty: Builder.getPtrTy(), ArraySize: Builder.getInt64(C: 1));
1903	LoadInst *Load1 = Builder.CreateLoad(Ty: Alloc1->getAllocatedType(), Ptr: Alloc1);
1904	MapVector<Value *, Value *> AlignedVars;
1905	AlignedVars.insert(KV: {Load1, Builder.getInt64(C: AlignmentValue)});
1906
1907	CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, UseIVBits: 32);
1908
1909	// Simd-ize the loop.
1910	OMPBuilder.applySimd(CLI, AlignedVars, /* IfCond */ nullptr,
1911	OrderKind::OMP_ORDER_unknown,
1912	/* Simdlen */ nullptr,
1913	/* Safelen */ nullptr);
1914
1915	OMPBuilder.finalize();
1916	EXPECT_FALSE(verifyModule(*M, &errs()));
1917
1918	PassBuilder PB;
1919	FunctionAnalysisManager FAM;
1920	PB.registerFunctionAnalyses(FAM);
1921	LoopInfo &LI = FAM.getResult<LoopAnalysis>(IR&: *F);
1922
1923	const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1924	EXPECT_EQ(TopLvl.size(), 1u);
1925
1926	Loop *L = TopLvl.front();
1927	EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1928	EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1929
1930	// Check for llvm.access.group metadata attached to the printf
1931	// function in the loop body.
1932	BasicBlock *LoopBody = CLI->getBody();
1933	EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
1934	return I.getMetadata("llvm.access.group") != nullptr;
1935	}));
1936
1937	// Check if number of assumption instructions is equal to number of aligned
1938	// variables
1939	BasicBlock *LoopPreheader = CLI->getPreheader();
1940	size_t NumAssummptionCallsInPreheader = count_if(
1941	Range&: *LoopPreheader, P: [](Instruction &I) { return isa<AssumeInst>(Val: I); });
1942	EXPECT_EQ(NumAssummptionCallsInPreheader, AlignedVars.size());
1943
1944	// Check if variables are correctly aligned
1945	for (Instruction &Instr : *LoopPreheader) {
1946	if (!isa<AssumeInst>(Val: Instr))
1947	continue;
1948	AssumeInst *AssumeInstruction = cast<AssumeInst>(Val: &Instr);
1949	if (AssumeInstruction->getNumTotalBundleOperands()) {
1950	auto Bundle = AssumeInstruction->getOperandBundleAt(Index: 0);
1951	if (Bundle.getTagName() == "align") {
1952	EXPECT_TRUE(isa<ConstantInt>(Bundle.Inputs[1]));
1953	auto ConstIntVal = dyn_cast<ConstantInt>(Val: Bundle.Inputs[1]);
1954	EXPECT_EQ(ConstIntVal->getSExtValue(), AlignmentValue);
1955	}
1956	}
1957	}
1958	}
1959	TEST_F(OpenMPIRBuilderTest, ApplySimdlen) {
1960	OpenMPIRBuilder OMPBuilder(*M);
1961	MapVector<Value *, Value *> AlignedVars;
1962	CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, UseIVBits: 32);
1963
1964	// Simd-ize the loop.
1965	OMPBuilder.applySimd(CLI, AlignedVars,
1966	/* IfCond */ nullptr, OrderKind::OMP_ORDER_unknown,
1967	ConstantInt::get(Type::getInt32Ty(Ctx), 3),
1968	/* Safelen */ nullptr);
1969
1970	OMPBuilder.finalize();
1971	EXPECT_FALSE(verifyModule(*M, &errs()));
1972
1973	PassBuilder PB;
1974	FunctionAnalysisManager FAM;
1975	PB.registerFunctionAnalyses(FAM);
1976	LoopInfo &LI = FAM.getResult<LoopAnalysis>(IR&: *F);
1977
1978	const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1979	EXPECT_EQ(TopLvl.size(), 1u);
1980
1981	Loop *L = TopLvl.front();
1982	EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1983	EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1984	EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);
1985
1986	// Check for llvm.access.group metadata attached to the printf
1987	// function in the loop body.
1988	BasicBlock *LoopBody = CLI->getBody();
1989	EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
1990	return I.getMetadata("llvm.access.group") != nullptr;
1991	}));
1992	}
1993
1994	TEST_F(OpenMPIRBuilderTest, ApplySafelenOrderConcurrent) {
1995	OpenMPIRBuilder OMPBuilder(*M);
1996	MapVector<Value *, Value *> AlignedVars;
1997
1998	CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, UseIVBits: 32);
1999
2000	// Simd-ize the loop.
2001	OMPBuilder.applySimd(
2002	CLI, AlignedVars, /* IfCond */ nullptr, OrderKind::OMP_ORDER_concurrent,
2003	/* Simdlen */ nullptr, ConstantInt::get(Type::getInt32Ty(Ctx), 3));
2004
2005	OMPBuilder.finalize();
2006	EXPECT_FALSE(verifyModule(*M, &errs()));
2007
2008	PassBuilder PB;
2009	FunctionAnalysisManager FAM;
2010	PB.registerFunctionAnalyses(FAM);
2011	LoopInfo &LI = FAM.getResult<LoopAnalysis>(IR&: *F);
2012
2013	const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
2014	EXPECT_EQ(TopLvl.size(), 1u);
2015
2016	Loop *L = TopLvl.front();
2017	// Parallel metadata shoudl be attached because of presence of
2018	// the order(concurrent) OpenMP clause
2019	EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
2020	EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
2021	EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);
2022
2023	// Check for llvm.access.group metadata attached to the printf
2024	// function in the loop body.
2025	BasicBlock *LoopBody = CLI->getBody();
2026	EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
2027	return I.getMetadata("llvm.access.group") != nullptr;
2028	}));
2029	}
2030
2031	TEST_F(OpenMPIRBuilderTest, ApplySafelen) {
2032	OpenMPIRBuilder OMPBuilder(*M);
2033	MapVector<Value *, Value *> AlignedVars;
2034
2035	CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, UseIVBits: 32);
2036
2037	OMPBuilder.applySimd(
2038	CLI, AlignedVars, /* IfCond */ nullptr, OrderKind::OMP_ORDER_unknown,
2039	/* Simdlen */ nullptr, ConstantInt::get(Type::getInt32Ty(Ctx), 3));
2040
2041	OMPBuilder.finalize();
2042	EXPECT_FALSE(verifyModule(*M, &errs()));
2043
2044	PassBuilder PB;
2045	FunctionAnalysisManager FAM;
2046	PB.registerFunctionAnalyses(FAM);
2047	LoopInfo &LI = FAM.getResult<LoopAnalysis>(IR&: *F);
2048
2049	const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
2050	EXPECT_EQ(TopLvl.size(), 1u);
2051
2052	Loop *L = TopLvl.front();
2053	EXPECT_FALSE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
2054	EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
2055	EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);
2056
2057	// Check for llvm.access.group metadata attached to the printf
2058	// function in the loop body.
2059	BasicBlock *LoopBody = CLI->getBody();
2060	EXPECT_FALSE(any_of(*LoopBody, [](Instruction &I) {
2061	return I.getMetadata("llvm.access.group") != nullptr;
2062	}));
2063	}
2064
2065	TEST_F(OpenMPIRBuilderTest, ApplySimdlenSafelen) {
2066	OpenMPIRBuilder OMPBuilder(*M);
2067	MapVector<Value *, Value *> AlignedVars;
2068
2069	CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, UseIVBits: 32);
2070
2071	OMPBuilder.applySimd(CLI, AlignedVars, /* IfCond */ nullptr,
2072	OrderKind::OMP_ORDER_unknown,
2073	ConstantInt::get(Type::getInt32Ty(Ctx), 2),
2074	ConstantInt::get(Type::getInt32Ty(Ctx), 3));
2075
2076	OMPBuilder.finalize();
2077	EXPECT_FALSE(verifyModule(*M, &errs()));
2078
2079	PassBuilder PB;
2080	FunctionAnalysisManager FAM;
2081	PB.registerFunctionAnalyses(FAM);
2082	LoopInfo &LI = FAM.getResult<LoopAnalysis>(IR&: *F);
2083
2084	const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
2085	EXPECT_EQ(TopLvl.size(), 1u);
2086
2087	Loop *L = TopLvl.front();
2088	EXPECT_FALSE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
2089	EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
2090	EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 2);
2091
2092	// Check for llvm.access.group metadata attached to the printf
2093	// function in the loop body.
2094	BasicBlock *LoopBody = CLI->getBody();
2095	EXPECT_FALSE(any_of(*LoopBody, [](Instruction &I) {
2096	return I.getMetadata("llvm.access.group") != nullptr;
2097	}));
2098	}
2099
2100	TEST_F(OpenMPIRBuilderTest, ApplySimdIf) {
2101	OpenMPIRBuilder OMPBuilder(*M);
2102	IRBuilder<> Builder(BB);
2103	MapVector<Value *, Value *> AlignedVars;
2104	AllocaInst *Alloc1 = Builder.CreateAlloca(Ty: Builder.getInt32Ty());
2105	AllocaInst *Alloc2 = Builder.CreateAlloca(Ty: Builder.getInt32Ty());
2106
2107	// Generation of if condition
2108	Builder.CreateStore(Val: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 0U), Ptr: Alloc1);
2109	Builder.CreateStore(Val: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 1U), Ptr: Alloc2);
2110	LoadInst *Load1 = Builder.CreateLoad(Ty: Alloc1->getAllocatedType(), Ptr: Alloc1);
2111	LoadInst *Load2 = Builder.CreateLoad(Ty: Alloc2->getAllocatedType(), Ptr: Alloc2);
2112
2113	Value *IfCmp = Builder.CreateICmpNE(LHS: Load1, RHS: Load2);
2114
2115	CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, UseIVBits: 32);
2116
2117	// Simd-ize the loop with if condition
2118	OMPBuilder.applySimd(CLI, AlignedVars, IfCmp, OrderKind::OMP_ORDER_unknown,
2119	ConstantInt::get(Type::getInt32Ty(Ctx), 3),
2120	/* Safelen */ nullptr);
2121
2122	OMPBuilder.finalize();
2123	EXPECT_FALSE(verifyModule(*M, &errs()));
2124
2125	PassBuilder PB;
2126	FunctionAnalysisManager FAM;
2127	PB.registerFunctionAnalyses(FAM);
2128	LoopInfo &LI = FAM.getResult<LoopAnalysis>(IR&: *F);
2129
2130	// Check if there are two loops (one with enabled vectorization)
2131	const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
2132	EXPECT_EQ(TopLvl.size(), 2u);
2133
2134	Loop *L = TopLvl[0];
2135	EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
2136	EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
2137	EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);
2138
2139	// The second loop should have disabled vectorization
2140	L = TopLvl[1];
2141	EXPECT_FALSE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
2142	EXPECT_FALSE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
2143	// Check for llvm.access.group metadata attached to the printf
2144	// function in the loop body.
2145	BasicBlock *LoopBody = CLI->getBody();
2146	EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
2147	return I.getMetadata("llvm.access.group") != nullptr;
2148	}));
2149	}
2150
2151	TEST_F(OpenMPIRBuilderTest, UnrollLoopFull) {
2152	OpenMPIRBuilder OMPBuilder(*M);
2153
2154	CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, UseIVBits: 32);
2155
2156	// Unroll the loop.
2157	OMPBuilder.unrollLoopFull(DL, Loop: CLI);
2158
2159	OMPBuilder.finalize();
2160	EXPECT_FALSE(verifyModule(*M, &errs()));
2161
2162	PassBuilder PB;
2163	FunctionAnalysisManager FAM;
2164	PB.registerFunctionAnalyses(FAM);
2165	LoopInfo &LI = FAM.getResult<LoopAnalysis>(IR&: *F);
2166
2167	const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
2168	EXPECT_EQ(TopLvl.size(), 1u);
2169
2170	Loop *L = TopLvl.front();
2171	EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.enable"));
2172	EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.full"));
2173	}
2174
2175	TEST_F(OpenMPIRBuilderTest, UnrollLoopPartial) {
2176	OpenMPIRBuilder OMPBuilder(*M);
2177	CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, UseIVBits: 32);
2178
2179	// Unroll the loop.
2180	CanonicalLoopInfo *UnrolledLoop = nullptr;
2181	OMPBuilder.unrollLoopPartial(DL, Loop: CLI, Factor: 5, UnrolledCLI: &UnrolledLoop);
2182	ASSERT_NE(UnrolledLoop, nullptr);
2183
2184	OMPBuilder.finalize();
2185	EXPECT_FALSE(verifyModule(*M, &errs()));
2186	UnrolledLoop->assertOK();
2187
2188	PassBuilder PB;
2189	FunctionAnalysisManager FAM;
2190	PB.registerFunctionAnalyses(FAM);
2191	LoopInfo &LI = FAM.getResult<LoopAnalysis>(IR&: *F);
2192
2193	const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
2194	EXPECT_EQ(TopLvl.size(), 1u);
2195	Loop *Outer = TopLvl.front();
2196	EXPECT_EQ(Outer->getHeader(), UnrolledLoop->getHeader());
2197	EXPECT_EQ(Outer->getLoopLatch(), UnrolledLoop->getLatch());
2198	EXPECT_EQ(Outer->getExitingBlock(), UnrolledLoop->getCond());
2199	EXPECT_EQ(Outer->getExitBlock(), UnrolledLoop->getExit());
2200
2201	EXPECT_EQ(Outer->getSubLoops().size(), 1u);
2202	Loop *Inner = Outer->getSubLoops().front();
2203
2204	EXPECT_TRUE(getBooleanLoopAttribute(Inner, "llvm.loop.unroll.enable"));
2205	EXPECT_EQ(getIntLoopAttribute(Inner, "llvm.loop.unroll.count"), 5);
2206	}
2207
2208	TEST_F(OpenMPIRBuilderTest, UnrollLoopHeuristic) {
2209	OpenMPIRBuilder OMPBuilder(*M);
2210
2211	CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, UseIVBits: 32);
2212
2213	// Unroll the loop.
2214	OMPBuilder.unrollLoopHeuristic(DL, Loop: CLI);
2215
2216	OMPBuilder.finalize();
2217	EXPECT_FALSE(verifyModule(*M, &errs()));
2218
2219	PassBuilder PB;
2220	FunctionAnalysisManager FAM;
2221	PB.registerFunctionAnalyses(FAM);
2222	LoopInfo &LI = FAM.getResult<LoopAnalysis>(IR&: *F);
2223
2224	const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
2225	EXPECT_EQ(TopLvl.size(), 1u);
2226
2227	Loop *L = TopLvl.front();
2228	EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.enable"));
2229	}
2230
2231	TEST_F(OpenMPIRBuilderTest, StaticWorkshareLoopTarget) {
2232	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2233	std::string oldDLStr = M->getDataLayoutStr();
2234	M->setDataLayout(
2235	"e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:"
2236	"256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:"
2237	"256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8");
2238	OpenMPIRBuilder OMPBuilder(*M);
2239	OMPBuilder.Config.IsTargetDevice = true;
2240	OMPBuilder.initialize();
2241	IRBuilder<> Builder(BB);
2242	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2243	InsertPointTy AllocaIP = Builder.saveIP();
2244
2245	Type *LCTy = Type::getInt32Ty(C&: Ctx);
2246	Value *StartVal = ConstantInt::get(Ty: LCTy, V: 10);
2247	Value *StopVal = ConstantInt::get(Ty: LCTy, V: 52);
2248	Value *StepVal = ConstantInt::get(Ty: LCTy, V: 2);
2249	auto LoopBodyGen = [&](InsertPointTy, Value *) {};
2250
2251	CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
2252	Loc, BodyGenCB: LoopBodyGen, Start: StartVal, Stop: StopVal, Step: StepVal, IsSigned: false, InclusiveStop: false);
2253	BasicBlock *Preheader = CLI->getPreheader();
2254	Value *TripCount = CLI->getTripCount();
2255
2256	Builder.SetInsertPoint(TheBB: BB, IP: BB->getFirstInsertionPt());
2257
2258	IRBuilder<>::InsertPoint AfterIP = OMPBuilder.applyWorkshareLoop(
2259	DL, CLI, AllocaIP, true, OMP_SCHEDULE_Static, nullptr, false, false,
2260	false, false, WorksharingLoopType::ForStaticLoop);
2261	Builder.restoreIP(IP: AfterIP);
2262	Builder.CreateRetVoid();
2263
2264	OMPBuilder.finalize();
2265	EXPECT_FALSE(verifyModule(*M, &errs()));
2266
2267	CallInst *WorkshareLoopRuntimeCall = nullptr;
2268	int WorkshareLoopRuntimeCallCnt = 0;
2269	for (auto Inst = Preheader->begin(); Inst != Preheader->end(); ++Inst) {
2270	CallInst *Call = dyn_cast<CallInst>(Val&: Inst);
2271	if (!Call)
2272	continue;
2273	if (!Call->getCalledFunction())
2274	continue;
2275
2276	if (Call->getCalledFunction()->getName() == "__kmpc_for_static_loop_4u") {
2277	WorkshareLoopRuntimeCall = Call;
2278	WorkshareLoopRuntimeCallCnt++;
2279	}
2280	}
2281	EXPECT_NE(WorkshareLoopRuntimeCall, nullptr);
2282	// Verify that there is only one call to workshare loop function
2283	EXPECT_EQ(WorkshareLoopRuntimeCallCnt, 1);
2284	// Check that pointer to loop body function is passed as second argument
2285	Value *LoopBodyFuncArg = WorkshareLoopRuntimeCall->getArgOperand(i: 1);
2286	EXPECT_EQ(Builder.getPtrTy(), LoopBodyFuncArg->getType());
2287	Function *ArgFunction = dyn_cast<Function>(Val: LoopBodyFuncArg);
2288	EXPECT_NE(ArgFunction, nullptr);
2289	EXPECT_EQ(ArgFunction->arg_size(), 1u);
2290	EXPECT_EQ(ArgFunction->getArg(0)->getType(), TripCount->getType());
2291	// Check that no variables except for loop counter are used in loop body
2292	EXPECT_EQ(Constant::getNullValue(Builder.getPtrTy()),
2293	WorkshareLoopRuntimeCall->getArgOperand(2));
2294	// Check loop trip count argument
2295	EXPECT_EQ(TripCount, WorkshareLoopRuntimeCall->getArgOperand(3));
2296	}
2297
2298	TEST_F(OpenMPIRBuilderTest, StaticWorkShareLoop) {
2299	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2300	OpenMPIRBuilder OMPBuilder(*M);
2301	OMPBuilder.Config.IsTargetDevice = false;
2302	OMPBuilder.initialize();
2303	IRBuilder<> Builder(BB);
2304	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2305
2306	Type *LCTy = Type::getInt32Ty(C&: Ctx);
2307	Value *StartVal = ConstantInt::get(Ty: LCTy, V: 10);
2308	Value *StopVal = ConstantInt::get(Ty: LCTy, V: 52);
2309	Value *StepVal = ConstantInt::get(Ty: LCTy, V: 2);
2310	auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
2311
2312	CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
2313	Loc, BodyGenCB: LoopBodyGen, Start: StartVal, Stop: StopVal, Step: StepVal,
2314	/*IsSigned=*/false, /*InclusiveStop=*/false);
2315	BasicBlock *Preheader = CLI->getPreheader();
2316	BasicBlock *Body = CLI->getBody();
2317	Value *IV = CLI->getIndVar();
2318	BasicBlock *ExitBlock = CLI->getExit();
2319
2320	Builder.SetInsertPoint(TheBB: BB, IP: BB->getFirstInsertionPt());
2321	InsertPointTy AllocaIP = Builder.saveIP();
2322
2323	OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true,
2324	OMP_SCHEDULE_Static);
2325
2326	BasicBlock *Cond = Body->getSinglePredecessor();
2327	Instruction *Cmp = &*Cond->begin();
2328	Value *TripCount = Cmp->getOperand(i: 1);
2329
2330	auto AllocaIter = BB->begin();
2331	ASSERT_GE(std::distance(BB->begin(), BB->end()), 4);
2332	AllocaInst *PLastIter = dyn_cast<AllocaInst>(Val: &*(AllocaIter++));
2333	AllocaInst *PLowerBound = dyn_cast<AllocaInst>(Val: &*(AllocaIter++));
2334	AllocaInst *PUpperBound = dyn_cast<AllocaInst>(Val: &*(AllocaIter++));
2335	AllocaInst *PStride = dyn_cast<AllocaInst>(Val: &*(AllocaIter++));
2336	EXPECT_NE(PLastIter, nullptr);
2337	EXPECT_NE(PLowerBound, nullptr);
2338	EXPECT_NE(PUpperBound, nullptr);
2339	EXPECT_NE(PStride, nullptr);
2340
2341	auto PreheaderIter = Preheader->begin();
2342	ASSERT_GE(std::distance(Preheader->begin(), Preheader->end()), 7);
2343	StoreInst *LowerBoundStore = dyn_cast<StoreInst>(Val: &*(PreheaderIter++));
2344	StoreInst *UpperBoundStore = dyn_cast<StoreInst>(Val: &*(PreheaderIter++));
2345	StoreInst *StrideStore = dyn_cast<StoreInst>(Val: &*(PreheaderIter++));
2346	ASSERT_NE(LowerBoundStore, nullptr);
2347	ASSERT_NE(UpperBoundStore, nullptr);
2348	ASSERT_NE(StrideStore, nullptr);
2349
2350	auto *OrigLowerBound =
2351	dyn_cast<ConstantInt>(Val: LowerBoundStore->getValueOperand());
2352	auto *OrigUpperBound =
2353	dyn_cast<ConstantInt>(Val: UpperBoundStore->getValueOperand());
2354	auto *OrigStride = dyn_cast<ConstantInt>(Val: StrideStore->getValueOperand());
2355	ASSERT_NE(OrigLowerBound, nullptr);
2356	ASSERT_NE(OrigUpperBound, nullptr);
2357	ASSERT_NE(OrigStride, nullptr);
2358	EXPECT_EQ(OrigLowerBound->getValue(), 0);
2359	EXPECT_EQ(OrigUpperBound->getValue(), 20);
2360	EXPECT_EQ(OrigStride->getValue(), 1);
2361
2362	// Check that the loop IV is updated to account for the lower bound returned
2363	// by the OpenMP runtime call.
2364	BinaryOperator *Add = dyn_cast<BinaryOperator>(Val: &Body->front());
2365	EXPECT_EQ(Add->getOperand(0), IV);
2366	auto *LoadedLowerBound = dyn_cast<LoadInst>(Val: Add->getOperand(i_nocapture: 1));
2367	ASSERT_NE(LoadedLowerBound, nullptr);
2368	EXPECT_EQ(LoadedLowerBound->getPointerOperand(), PLowerBound);
2369
2370	// Check that the trip count is updated to account for the lower and upper
2371	// bounds return by the OpenMP runtime call.
2372	auto *AddOne = dyn_cast<Instruction>(Val: TripCount);
2373	ASSERT_NE(AddOne, nullptr);
2374	ASSERT_TRUE(AddOne->isBinaryOp());
2375	auto *One = dyn_cast<ConstantInt>(Val: AddOne->getOperand(i: 1));
2376	ASSERT_NE(One, nullptr);
2377	EXPECT_EQ(One->getValue(), 1);
2378	auto *Difference = dyn_cast<Instruction>(Val: AddOne->getOperand(i: 0));
2379	ASSERT_NE(Difference, nullptr);
2380	ASSERT_TRUE(Difference->isBinaryOp());
2381	EXPECT_EQ(Difference->getOperand(1), LoadedLowerBound);
2382	auto *LoadedUpperBound = dyn_cast<LoadInst>(Val: Difference->getOperand(i: 0));
2383	ASSERT_NE(LoadedUpperBound, nullptr);
2384	EXPECT_EQ(LoadedUpperBound->getPointerOperand(), PUpperBound);
2385
2386	// The original loop iterator should only be used in the condition, in the
2387	// increment and in the statement that adds the lower bound to it.
2388	EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
2389
2390	// The exit block should contain the "fini" call and the barrier call,
2391	// plus the call to obtain the thread ID.
2392	size_t NumCallsInExitBlock =
2393	count_if(Range&: *ExitBlock, P: [](Instruction &I) { return isa<CallInst>(Val: I); });
2394	EXPECT_EQ(NumCallsInExitBlock, 3u);
2395	}
2396
2397	TEST_P(OpenMPIRBuilderTestWithIVBits, StaticChunkedWorkshareLoop) {
2398	unsigned IVBits = GetParam();
2399
2400	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2401	OpenMPIRBuilder OMPBuilder(*M);
2402	OMPBuilder.Config.IsTargetDevice = false;
2403
2404	BasicBlock *Body;
2405	CallInst *Call;
2406	CanonicalLoopInfo *CLI =
2407	buildSingleLoopFunction(DL, OMPBuilder, UseIVBits: IVBits, Call: &Call, BodyCode: &Body);
2408
2409	Instruction *OrigIndVar = CLI->getIndVar();
2410	EXPECT_EQ(Call->getOperand(1), OrigIndVar);
2411
2412	Type *LCTy = Type::getInt32Ty(C&: Ctx);
2413	Value *ChunkSize = ConstantInt::get(Ty: LCTy, V: 5);
2414	InsertPointTy AllocaIP{&F->getEntryBlock(),
2415	F->getEntryBlock().getFirstInsertionPt()};
2416	OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true,
2417	OMP_SCHEDULE_Static, ChunkSize);
2418
2419	OMPBuilder.finalize();
2420	EXPECT_FALSE(verifyModule(*M, &errs()));
2421
2422	BasicBlock *Entry = &F->getEntryBlock();
2423	BasicBlock *Preheader = Entry->getSingleSuccessor();
2424
2425	BasicBlock *DispatchPreheader = Preheader->getSingleSuccessor();
2426	BasicBlock *DispatchHeader = DispatchPreheader->getSingleSuccessor();
2427	BasicBlock *DispatchCond = DispatchHeader->getSingleSuccessor();
2428	BasicBlock *DispatchBody = succ_begin(BB: DispatchCond)[0];
2429	BasicBlock *DispatchExit = succ_begin(BB: DispatchCond)[1];
2430	BasicBlock *DispatchAfter = DispatchExit->getSingleSuccessor();
2431	BasicBlock *Return = DispatchAfter->getSingleSuccessor();
2432
2433	BasicBlock *ChunkPreheader = DispatchBody->getSingleSuccessor();
2434	BasicBlock *ChunkHeader = ChunkPreheader->getSingleSuccessor();
2435	BasicBlock *ChunkCond = ChunkHeader->getSingleSuccessor();
2436	BasicBlock *ChunkBody = succ_begin(BB: ChunkCond)[0];
2437	BasicBlock *ChunkExit = succ_begin(BB: ChunkCond)[1];
2438	BasicBlock *ChunkInc = ChunkBody->getSingleSuccessor();
2439	BasicBlock *ChunkAfter = ChunkExit->getSingleSuccessor();
2440
2441	BasicBlock *DispatchInc = ChunkAfter;
2442
2443	EXPECT_EQ(ChunkBody, Body);
2444	EXPECT_EQ(ChunkInc->getSingleSuccessor(), ChunkHeader);
2445	EXPECT_EQ(DispatchInc->getSingleSuccessor(), DispatchHeader);
2446
2447	EXPECT_TRUE(isa<ReturnInst>(Return->front()));
2448
2449	Value *NewIV = Call->getOperand(i_nocapture: 1);
2450	EXPECT_EQ(NewIV->getType()->getScalarSizeInBits(), IVBits);
2451
2452	CallInst *InitCall = findSingleCall(
2453	F,
2454	FnID: (IVBits > 32) ? omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u
2455	: omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u,
2456	OMPBuilder);
2457	EXPECT_EQ(InitCall->getParent(), Preheader);
2458	EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(2))->getSExtValue(), 33);
2459	EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(7))->getSExtValue(), 1);
2460	EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(8))->getSExtValue(), 5);
2461
2462	CallInst *FiniCall = findSingleCall(
2463	F, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_fini, OMPBuilder);
2464	EXPECT_EQ(FiniCall->getParent(), DispatchExit);
2465
2466	CallInst *BarrierCall = findSingleCall(
2467	F, FnID: omp::RuntimeFunction::OMPRTL___kmpc_barrier, OMPBuilder);
2468	EXPECT_EQ(BarrierCall->getParent(), DispatchExit);
2469	}
2470
2471	INSTANTIATE_TEST_SUITE_P(IVBits, OpenMPIRBuilderTestWithIVBits,
2472	::testing::Values(8, 16, 32, 64));
2473
2474	TEST_P(OpenMPIRBuilderTestWithParams, DynamicWorkShareLoop) {
2475	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2476	OpenMPIRBuilder OMPBuilder(*M);
2477	OMPBuilder.Config.IsTargetDevice = false;
2478	OMPBuilder.initialize();
2479	IRBuilder<> Builder(BB);
2480	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2481
2482	omp::OMPScheduleType SchedType = GetParam();
2483	uint32_t ChunkSize = 1;
2484	switch (SchedType & ~OMPScheduleType::ModifierMask) {
2485	case omp::OMPScheduleType::BaseDynamicChunked:
2486	case omp::OMPScheduleType::BaseGuidedChunked:
2487	ChunkSize = 7;
2488	break;
2489	case omp::OMPScheduleType::BaseAuto:
2490	case omp::OMPScheduleType::BaseRuntime:
2491	ChunkSize = 1;
2492	break;
2493	default:
2494	assert(0 && "unknown type for this test");
2495	break;
2496	}
2497
2498	Type *LCTy = Type::getInt32Ty(C&: Ctx);
2499	Value *StartVal = ConstantInt::get(Ty: LCTy, V: 10);
2500	Value *StopVal = ConstantInt::get(Ty: LCTy, V: 52);
2501	Value *StepVal = ConstantInt::get(Ty: LCTy, V: 2);
2502	Value *ChunkVal =
2503	(ChunkSize == 1) ? nullptr : ConstantInt::get(Ty: LCTy, V: ChunkSize);
2504	auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
2505
2506	CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
2507	Loc, BodyGenCB: LoopBodyGen, Start: StartVal, Stop: StopVal, Step: StepVal,
2508	/*IsSigned=*/false, /*InclusiveStop=*/false);
2509
2510	Builder.SetInsertPoint(TheBB: BB, IP: BB->getFirstInsertionPt());
2511	InsertPointTy AllocaIP = Builder.saveIP();
2512
2513	// Collect all the info from CLI, as it isn't usable after the call to
2514	// createDynamicWorkshareLoop.
2515	InsertPointTy AfterIP = CLI->getAfterIP();
2516	BasicBlock *Preheader = CLI->getPreheader();
2517	BasicBlock *ExitBlock = CLI->getExit();
2518	BasicBlock *LatchBlock = CLI->getLatch();
2519	Value *IV = CLI->getIndVar();
2520
2521	InsertPointTy EndIP = OMPBuilder.applyWorkshareLoop(
2522	DL, CLI, AllocaIP, /*NeedsBarrier=*/true, getSchedKind(SchedType),
2523	ChunkVal, /*Simd=*/false,
2524	(SchedType & omp::OMPScheduleType::ModifierMonotonic) ==
2525	omp::OMPScheduleType::ModifierMonotonic,
2526	(SchedType & omp::OMPScheduleType::ModifierNonmonotonic) ==
2527	omp::OMPScheduleType::ModifierNonmonotonic,
2528	/*Ordered=*/false);
2529
2530	// The returned value should be the "after" point.
2531	ASSERT_EQ(EndIP.getBlock(), AfterIP.getBlock());
2532	ASSERT_EQ(EndIP.getPoint(), AfterIP.getPoint());
2533
2534	auto AllocaIter = BB->begin();
2535	ASSERT_GE(std::distance(BB->begin(), BB->end()), 4);
2536	AllocaInst *PLastIter = dyn_cast<AllocaInst>(Val: &*(AllocaIter++));
2537	AllocaInst *PLowerBound = dyn_cast<AllocaInst>(Val: &*(AllocaIter++));
2538	AllocaInst *PUpperBound = dyn_cast<AllocaInst>(Val: &*(AllocaIter++));
2539	AllocaInst *PStride = dyn_cast<AllocaInst>(Val: &*(AllocaIter++));
2540	EXPECT_NE(PLastIter, nullptr);
2541	EXPECT_NE(PLowerBound, nullptr);
2542	EXPECT_NE(PUpperBound, nullptr);
2543	EXPECT_NE(PStride, nullptr);
2544
2545	auto PreheaderIter = Preheader->begin();
2546	ASSERT_GE(std::distance(Preheader->begin(), Preheader->end()), 6);
2547	StoreInst *LowerBoundStore = dyn_cast<StoreInst>(Val: &*(PreheaderIter++));
2548	StoreInst *UpperBoundStore = dyn_cast<StoreInst>(Val: &*(PreheaderIter++));
2549	StoreInst *StrideStore = dyn_cast<StoreInst>(Val: &*(PreheaderIter++));
2550	ASSERT_NE(LowerBoundStore, nullptr);
2551	ASSERT_NE(UpperBoundStore, nullptr);
2552	ASSERT_NE(StrideStore, nullptr);
2553
2554	CallInst *ThreadIdCall = dyn_cast<CallInst>(Val: &*(PreheaderIter++));
2555	ASSERT_NE(ThreadIdCall, nullptr);
2556	EXPECT_EQ(ThreadIdCall->getCalledFunction()->getName(),
2557	"__kmpc_global_thread_num");
2558
2559	CallInst *InitCall = dyn_cast<CallInst>(Val: &*PreheaderIter);
2560
2561	ASSERT_NE(InitCall, nullptr);
2562	EXPECT_EQ(InitCall->getCalledFunction()->getName(),
2563	"__kmpc_dispatch_init_4u");
2564	EXPECT_EQ(InitCall->arg_size(), 7U);
2565	EXPECT_EQ(InitCall->getArgOperand(6), ConstantInt::get(LCTy, ChunkSize));
2566	ConstantInt *SchedVal = cast<ConstantInt>(Val: InitCall->getArgOperand(i: 2));
2567	if ((SchedType & OMPScheduleType::MonotonicityMask) ==
2568	OMPScheduleType::None) {
2569	// Implementation is allowed to add default nonmonotonicity flag
2570	EXPECT_EQ(
2571	static_cast<OMPScheduleType>(SchedVal->getValue().getZExtValue()) |
2572	OMPScheduleType::ModifierNonmonotonic,
2573	SchedType | OMPScheduleType::ModifierNonmonotonic);
2574	} else {
2575	EXPECT_EQ(static_cast<OMPScheduleType>(SchedVal->getValue().getZExtValue()),
2576	SchedType);
2577	}
2578
2579	ConstantInt *OrigLowerBound =
2580	dyn_cast<ConstantInt>(Val: LowerBoundStore->getValueOperand());
2581	ConstantInt *OrigUpperBound =
2582	dyn_cast<ConstantInt>(Val: UpperBoundStore->getValueOperand());
2583	ConstantInt *OrigStride =
2584	dyn_cast<ConstantInt>(Val: StrideStore->getValueOperand());
2585	ASSERT_NE(OrigLowerBound, nullptr);
2586	ASSERT_NE(OrigUpperBound, nullptr);
2587	ASSERT_NE(OrigStride, nullptr);
2588	EXPECT_EQ(OrigLowerBound->getValue(), 1);
2589	EXPECT_EQ(OrigUpperBound->getValue(), 21);
2590	EXPECT_EQ(OrigStride->getValue(), 1);
2591
2592	CallInst *FiniCall = dyn_cast<CallInst>(
2593	Val: &*(LatchBlock->getTerminator()->getPrevNonDebugInstruction(SkipPseudoOp: true)));
2594	EXPECT_EQ(FiniCall, nullptr);
2595
2596	// The original loop iterator should only be used in the condition, in the
2597	// increment and in the statement that adds the lower bound to it.
2598	EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
2599
2600	// The exit block should contain the barrier call, plus the call to obtain
2601	// the thread ID.
2602	size_t NumCallsInExitBlock =
2603	count_if(Range&: *ExitBlock, P: [](Instruction &I) { return isa<CallInst>(Val: I); });
2604	EXPECT_EQ(NumCallsInExitBlock, 2u);
2605
2606	// Add a termination to our block and check that it is internally consistent.
2607	Builder.restoreIP(IP: EndIP);
2608	Builder.CreateRetVoid();
2609	OMPBuilder.finalize();
2610	EXPECT_FALSE(verifyModule(*M, &errs()));
2611	}
2612
2613	INSTANTIATE_TEST_SUITE_P(
2614	OpenMPWSLoopSchedulingTypes, OpenMPIRBuilderTestWithParams,
2615	::testing::Values(omp::OMPScheduleType::UnorderedDynamicChunked,
2616	omp::OMPScheduleType::UnorderedGuidedChunked,
2617	omp::OMPScheduleType::UnorderedAuto,
2618	omp::OMPScheduleType::UnorderedRuntime,
2619	omp::OMPScheduleType::UnorderedDynamicChunked |
2620	omp::OMPScheduleType::ModifierMonotonic,
2621	omp::OMPScheduleType::UnorderedDynamicChunked |
2622	omp::OMPScheduleType::ModifierNonmonotonic,
2623	omp::OMPScheduleType::UnorderedGuidedChunked |
2624	omp::OMPScheduleType::ModifierMonotonic,
2625	omp::OMPScheduleType::UnorderedGuidedChunked |
2626	omp::OMPScheduleType::ModifierNonmonotonic,
2627	omp::OMPScheduleType::UnorderedAuto |
2628	omp::OMPScheduleType::ModifierMonotonic,
2629	omp::OMPScheduleType::UnorderedRuntime |
2630	omp::OMPScheduleType::ModifierMonotonic));
2631
2632	TEST_F(OpenMPIRBuilderTest, DynamicWorkShareLoopOrdered) {
2633	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2634	OpenMPIRBuilder OMPBuilder(*M);
2635	OMPBuilder.Config.IsTargetDevice = false;
2636	OMPBuilder.initialize();
2637	IRBuilder<> Builder(BB);
2638	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2639
2640	uint32_t ChunkSize = 1;
2641	Type *LCTy = Type::getInt32Ty(C&: Ctx);
2642	Value *StartVal = ConstantInt::get(Ty: LCTy, V: 10);
2643	Value *StopVal = ConstantInt::get(Ty: LCTy, V: 52);
2644	Value *StepVal = ConstantInt::get(Ty: LCTy, V: 2);
2645	Value *ChunkVal = ConstantInt::get(Ty: LCTy, V: ChunkSize);
2646	auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
2647
2648	CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
2649	Loc, BodyGenCB: LoopBodyGen, Start: StartVal, Stop: StopVal, Step: StepVal,
2650	/*IsSigned=*/false, /*InclusiveStop=*/false);
2651
2652	Builder.SetInsertPoint(TheBB: BB, IP: BB->getFirstInsertionPt());
2653	InsertPointTy AllocaIP = Builder.saveIP();
2654
2655	// Collect all the info from CLI, as it isn't usable after the call to
2656	// createDynamicWorkshareLoop.
2657	BasicBlock *Preheader = CLI->getPreheader();
2658	BasicBlock *ExitBlock = CLI->getExit();
2659	BasicBlock *LatchBlock = CLI->getLatch();
2660	Value *IV = CLI->getIndVar();
2661
2662	InsertPointTy EndIP = OMPBuilder.applyWorkshareLoop(
2663	DL, CLI, AllocaIP, /*NeedsBarrier=*/true, OMP_SCHEDULE_Static, ChunkVal,
2664	/*HasSimdModifier=*/false, /*HasMonotonicModifier=*/false,
2665	/*HasNonmonotonicModifier=*/false,
2666	/*HasOrderedClause=*/true);
2667
2668	// Add a termination to our block and check that it is internally consistent.
2669	Builder.restoreIP(IP: EndIP);
2670	Builder.CreateRetVoid();
2671	OMPBuilder.finalize();
2672	EXPECT_FALSE(verifyModule(*M, &errs()));
2673
2674	CallInst *InitCall = nullptr;
2675	for (Instruction &EI : *Preheader) {
2676	Instruction *Cur = &EI;
2677	if (isa<CallInst>(Val: Cur)) {
2678	InitCall = cast<CallInst>(Val: Cur);
2679	if (InitCall->getCalledFunction()->getName() == "__kmpc_dispatch_init_4u")
2680	break;
2681	InitCall = nullptr;
2682	}
2683	}
2684	EXPECT_NE(InitCall, nullptr);
2685	EXPECT_EQ(InitCall->arg_size(), 7U);
2686	ConstantInt *SchedVal = cast<ConstantInt>(Val: InitCall->getArgOperand(i: 2));
2687	EXPECT_EQ(SchedVal->getValue(),
2688	static_cast<uint64_t>(OMPScheduleType::OrderedStaticChunked));
2689
2690	CallInst *FiniCall = dyn_cast<CallInst>(
2691	Val: &*(LatchBlock->getTerminator()->getPrevNonDebugInstruction(SkipPseudoOp: true)));
2692	ASSERT_NE(FiniCall, nullptr);
2693	EXPECT_EQ(FiniCall->getCalledFunction()->getName(),
2694	"__kmpc_dispatch_fini_4u");
2695	EXPECT_EQ(FiniCall->arg_size(), 2U);
2696	EXPECT_EQ(InitCall->getArgOperand(0), FiniCall->getArgOperand(0));
2697	EXPECT_EQ(InitCall->getArgOperand(1), FiniCall->getArgOperand(1));
2698
2699	// The original loop iterator should only be used in the condition, in the
2700	// increment and in the statement that adds the lower bound to it.
2701	EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
2702
2703	// The exit block should contain the barrier call, plus the call to obtain
2704	// the thread ID.
2705	size_t NumCallsInExitBlock =
2706	count_if(Range&: *ExitBlock, P: [](Instruction &I) { return isa<CallInst>(Val: I); });
2707	EXPECT_EQ(NumCallsInExitBlock, 2u);
2708	}
2709
2710	TEST_F(OpenMPIRBuilderTest, MasterDirective) {
2711	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2712	OpenMPIRBuilder OMPBuilder(*M);
2713	OMPBuilder.initialize();
2714	F->setName("func");
2715	IRBuilder<> Builder(BB);
2716
2717	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2718
2719	AllocaInst *PrivAI = nullptr;
2720
2721	BasicBlock *EntryBB = nullptr;
2722	BasicBlock *ThenBB = nullptr;
2723
2724	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2725	if (AllocaIP.isSet())
2726	Builder.restoreIP(IP: AllocaIP);
2727	else
2728	Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
2729	PrivAI = Builder.CreateAlloca(Ty: F->arg_begin()->getType());
2730	Builder.CreateStore(Val: F->arg_begin(), Ptr: PrivAI);
2731
2732	llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2733	llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2734	EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2735
2736	Builder.restoreIP(IP: CodeGenIP);
2737
2738	// collect some info for checks later
2739	ThenBB = Builder.GetInsertBlock();
2740	EntryBB = ThenBB->getUniquePredecessor();
2741
2742	// simple instructions for body
2743	Value *PrivLoad =
2744	Builder.CreateLoad(Ty: PrivAI->getAllocatedType(), Ptr: PrivAI, Name: "local.use");
2745	Builder.CreateICmpNE(LHS: F->arg_begin(), RHS: PrivLoad);
2746	};
2747
2748	auto FiniCB = [&](InsertPointTy IP) {
2749	BasicBlock *IPBB = IP.getBlock();
2750	EXPECT_NE(IPBB->end(), IP.getPoint());
2751	};
2752
2753	Builder.restoreIP(IP: OMPBuilder.createMaster(Loc: Builder, BodyGenCB, FiniCB));
2754	Value *EntryBBTI = EntryBB->getTerminator();
2755	EXPECT_NE(EntryBBTI, nullptr);
2756	EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
2757	BranchInst *EntryBr = cast<BranchInst>(Val: EntryBB->getTerminator());
2758	EXPECT_TRUE(EntryBr->isConditional());
2759	EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
2760	BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
2761	EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
2762
2763	CmpInst *CondInst = cast<CmpInst>(Val: EntryBr->getCondition());
2764	EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
2765
2766	CallInst *MasterEntryCI = cast<CallInst>(Val: CondInst->getOperand(i_nocapture: 0));
2767	EXPECT_EQ(MasterEntryCI->arg_size(), 2U);
2768	EXPECT_EQ(MasterEntryCI->getCalledFunction()->getName(), "__kmpc_master");
2769	EXPECT_TRUE(isa<GlobalVariable>(MasterEntryCI->getArgOperand(0)));
2770
2771	CallInst *MasterEndCI = nullptr;
2772	for (auto &FI : *ThenBB) {
2773	Instruction *cur = &FI;
2774	if (isa<CallInst>(Val: cur)) {
2775	MasterEndCI = cast<CallInst>(Val: cur);
2776	if (MasterEndCI->getCalledFunction()->getName() == "__kmpc_end_master")
2777	break;
2778	MasterEndCI = nullptr;
2779	}
2780	}
2781	EXPECT_NE(MasterEndCI, nullptr);
2782	EXPECT_EQ(MasterEndCI->arg_size(), 2U);
2783	EXPECT_TRUE(isa<GlobalVariable>(MasterEndCI->getArgOperand(0)));
2784	EXPECT_EQ(MasterEndCI->getArgOperand(1), MasterEntryCI->getArgOperand(1));
2785	}
2786
2787	TEST_F(OpenMPIRBuilderTest, MaskedDirective) {
2788	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2789	OpenMPIRBuilder OMPBuilder(*M);
2790	OMPBuilder.initialize();
2791	F->setName("func");
2792	IRBuilder<> Builder(BB);
2793
2794	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2795
2796	AllocaInst *PrivAI = nullptr;
2797
2798	BasicBlock *EntryBB = nullptr;
2799	BasicBlock *ThenBB = nullptr;
2800
2801	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2802	if (AllocaIP.isSet())
2803	Builder.restoreIP(IP: AllocaIP);
2804	else
2805	Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
2806	PrivAI = Builder.CreateAlloca(Ty: F->arg_begin()->getType());
2807	Builder.CreateStore(Val: F->arg_begin(), Ptr: PrivAI);
2808
2809	llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2810	llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2811	EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2812
2813	Builder.restoreIP(IP: CodeGenIP);
2814
2815	// collect some info for checks later
2816	ThenBB = Builder.GetInsertBlock();
2817	EntryBB = ThenBB->getUniquePredecessor();
2818
2819	// simple instructions for body
2820	Value *PrivLoad =
2821	Builder.CreateLoad(Ty: PrivAI->getAllocatedType(), Ptr: PrivAI, Name: "local.use");
2822	Builder.CreateICmpNE(LHS: F->arg_begin(), RHS: PrivLoad);
2823	};
2824
2825	auto FiniCB = [&](InsertPointTy IP) {
2826	BasicBlock *IPBB = IP.getBlock();
2827	EXPECT_NE(IPBB->end(), IP.getPoint());
2828	};
2829
2830	Constant *Filter = ConstantInt::get(Ty: Type::getInt32Ty(C&: M->getContext()), V: 0);
2831	Builder.restoreIP(
2832	IP: OMPBuilder.createMasked(Loc: Builder, BodyGenCB, FiniCB, Filter));
2833	Value *EntryBBTI = EntryBB->getTerminator();
2834	EXPECT_NE(EntryBBTI, nullptr);
2835	EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
2836	BranchInst *EntryBr = cast<BranchInst>(Val: EntryBB->getTerminator());
2837	EXPECT_TRUE(EntryBr->isConditional());
2838	EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
2839	BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
2840	EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
2841
2842	CmpInst *CondInst = cast<CmpInst>(Val: EntryBr->getCondition());
2843	EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
2844
2845	CallInst *MaskedEntryCI = cast<CallInst>(Val: CondInst->getOperand(i_nocapture: 0));
2846	EXPECT_EQ(MaskedEntryCI->arg_size(), 3U);
2847	EXPECT_EQ(MaskedEntryCI->getCalledFunction()->getName(), "__kmpc_masked");
2848	EXPECT_TRUE(isa<GlobalVariable>(MaskedEntryCI->getArgOperand(0)));
2849
2850	CallInst *MaskedEndCI = nullptr;
2851	for (auto &FI : *ThenBB) {
2852	Instruction *cur = &FI;
2853	if (isa<CallInst>(Val: cur)) {
2854	MaskedEndCI = cast<CallInst>(Val: cur);
2855	if (MaskedEndCI->getCalledFunction()->getName() == "__kmpc_end_masked")
2856	break;
2857	MaskedEndCI = nullptr;
2858	}
2859	}
2860	EXPECT_NE(MaskedEndCI, nullptr);
2861	EXPECT_EQ(MaskedEndCI->arg_size(), 2U);
2862	EXPECT_TRUE(isa<GlobalVariable>(MaskedEndCI->getArgOperand(0)));
2863	EXPECT_EQ(MaskedEndCI->getArgOperand(1), MaskedEntryCI->getArgOperand(1));
2864	}
2865
2866	TEST_F(OpenMPIRBuilderTest, CriticalDirective) {
2867	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2868	OpenMPIRBuilder OMPBuilder(*M);
2869	OMPBuilder.initialize();
2870	F->setName("func");
2871	IRBuilder<> Builder(BB);
2872
2873	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2874
2875	AllocaInst *PrivAI = Builder.CreateAlloca(Ty: F->arg_begin()->getType());
2876
2877	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2878	// actual start for bodyCB
2879	llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2880	llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2881	EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2882
2883	// body begin
2884	Builder.restoreIP(IP: CodeGenIP);
2885	Builder.CreateStore(Val: F->arg_begin(), Ptr: PrivAI);
2886	Value *PrivLoad =
2887	Builder.CreateLoad(Ty: PrivAI->getAllocatedType(), Ptr: PrivAI, Name: "local.use");
2888	Builder.CreateICmpNE(LHS: F->arg_begin(), RHS: PrivLoad);
2889	};
2890
2891	auto FiniCB = [&](InsertPointTy IP) {
2892	BasicBlock *IPBB = IP.getBlock();
2893	EXPECT_NE(IPBB->end(), IP.getPoint());
2894	};
2895	BasicBlock *EntryBB = Builder.GetInsertBlock();
2896
2897	Builder.restoreIP(IP: OMPBuilder.createCritical(Loc: Builder, BodyGenCB, FiniCB,
2898	CriticalName: "testCRT", HintInst: nullptr));
2899
2900	CallInst *CriticalEntryCI = nullptr;
2901	for (auto &EI : *EntryBB) {
2902	Instruction *cur = &EI;
2903	if (isa<CallInst>(Val: cur)) {
2904	CriticalEntryCI = cast<CallInst>(Val: cur);
2905	if (CriticalEntryCI->getCalledFunction()->getName() == "__kmpc_critical")
2906	break;
2907	CriticalEntryCI = nullptr;
2908	}
2909	}
2910	EXPECT_NE(CriticalEntryCI, nullptr);
2911	EXPECT_EQ(CriticalEntryCI->arg_size(), 3U);
2912	EXPECT_EQ(CriticalEntryCI->getCalledFunction()->getName(), "__kmpc_critical");
2913	EXPECT_TRUE(isa<GlobalVariable>(CriticalEntryCI->getArgOperand(0)));
2914
2915	CallInst *CriticalEndCI = nullptr;
2916	for (auto &FI : *EntryBB) {
2917	Instruction *cur = &FI;
2918	if (isa<CallInst>(Val: cur)) {
2919	CriticalEndCI = cast<CallInst>(Val: cur);
2920	if (CriticalEndCI->getCalledFunction()->getName() ==
2921	"__kmpc_end_critical")
2922	break;
2923	CriticalEndCI = nullptr;
2924	}
2925	}
2926	EXPECT_NE(CriticalEndCI, nullptr);
2927	EXPECT_EQ(CriticalEndCI->arg_size(), 3U);
2928	EXPECT_TRUE(isa<GlobalVariable>(CriticalEndCI->getArgOperand(0)));
2929	EXPECT_EQ(CriticalEndCI->getArgOperand(1), CriticalEntryCI->getArgOperand(1));
2930	PointerType *CriticalNamePtrTy =
2931	PointerType::getUnqual(ElementType: ArrayType::get(ElementType: Type::getInt32Ty(C&: Ctx), NumElements: 8));
2932	EXPECT_EQ(CriticalEndCI->getArgOperand(2), CriticalEntryCI->getArgOperand(2));
2933	GlobalVariable *GV =
2934	dyn_cast<GlobalVariable>(Val: CriticalEndCI->getArgOperand(i: 2));
2935	ASSERT_NE(GV, nullptr);
2936	EXPECT_EQ(GV->getType(), CriticalNamePtrTy);
2937	const DataLayout &DL = M->getDataLayout();
2938	const llvm::Align TypeAlign = DL.getABITypeAlign(Ty: CriticalNamePtrTy);
2939	const llvm::Align PtrAlign = DL.getPointerABIAlignment(AS: GV->getAddressSpace());
2940	if (const llvm::MaybeAlign Alignment = GV->getAlign())
2941	EXPECT_EQ(*Alignment, std::max(TypeAlign, PtrAlign));
2942	}
2943
2944	TEST_F(OpenMPIRBuilderTest, OrderedDirectiveDependSource) {
2945	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2946	OpenMPIRBuilder OMPBuilder(*M);
2947	OMPBuilder.initialize();
2948	F->setName("func");
2949	IRBuilder<> Builder(BB);
2950	LLVMContext &Ctx = M->getContext();
2951
2952	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2953
2954	InsertPointTy AllocaIP(&F->getEntryBlock(),
2955	F->getEntryBlock().getFirstInsertionPt());
2956
2957	unsigned NumLoops = 2;
2958	SmallVector<Value *, 2> StoreValues;
2959	Type *LCTy = Type::getInt64Ty(C&: Ctx);
2960	StoreValues.emplace_back(Args: ConstantInt::get(Ty: LCTy, V: 1));
2961	StoreValues.emplace_back(Args: ConstantInt::get(Ty: LCTy, V: 2));
2962
2963	// Test for "#omp ordered depend(source)"
2964	Builder.restoreIP(IP: OMPBuilder.createOrderedDepend(Loc: Builder, AllocaIP, NumLoops,
2965	StoreValues, Name: ".cnt.addr",
2966	/*IsDependSource=*/true));
2967
2968	Builder.CreateRetVoid();
2969	OMPBuilder.finalize();
2970	EXPECT_FALSE(verifyModule(*M, &errs()));
2971
2972	AllocaInst *AllocInst = dyn_cast<AllocaInst>(Val: &BB->front());
2973	ASSERT_NE(AllocInst, nullptr);
2974	ArrayType *ArrType = dyn_cast<ArrayType>(Val: AllocInst->getAllocatedType());
2975	EXPECT_EQ(ArrType->getNumElements(), NumLoops);
2976	EXPECT_TRUE(
2977	AllocInst->getAllocatedType()->getArrayElementType()->isIntegerTy(64));
2978
2979	Instruction *IterInst = dyn_cast<Instruction>(Val: AllocInst);
2980	for (unsigned Iter = 0; Iter < NumLoops; Iter++) {
2981	GetElementPtrInst *DependAddrGEPIter =
2982	dyn_cast<GetElementPtrInst>(Val: IterInst->getNextNode());
2983	ASSERT_NE(DependAddrGEPIter, nullptr);
2984	EXPECT_EQ(DependAddrGEPIter->getPointerOperand(), AllocInst);
2985	EXPECT_EQ(DependAddrGEPIter->getNumIndices(), (unsigned)2);
2986	auto *FirstIdx = dyn_cast<ConstantInt>(Val: DependAddrGEPIter->getOperand(i_nocapture: 1));
2987	auto *SecondIdx = dyn_cast<ConstantInt>(Val: DependAddrGEPIter->getOperand(i_nocapture: 2));
2988	ASSERT_NE(FirstIdx, nullptr);
2989	ASSERT_NE(SecondIdx, nullptr);
2990	EXPECT_EQ(FirstIdx->getValue(), 0);
2991	EXPECT_EQ(SecondIdx->getValue(), Iter);
2992	StoreInst *StoreValue =
2993	dyn_cast<StoreInst>(Val: DependAddrGEPIter->getNextNode());
2994	ASSERT_NE(StoreValue, nullptr);
2995	EXPECT_EQ(StoreValue->getValueOperand(), StoreValues[Iter]);
2996	EXPECT_EQ(StoreValue->getPointerOperand(), DependAddrGEPIter);
2997	EXPECT_EQ(StoreValue->getAlign(), Align(8));
2998	IterInst = dyn_cast<Instruction>(Val: StoreValue);
2999	}
3000
3001	GetElementPtrInst *DependBaseAddrGEP =
3002	dyn_cast<GetElementPtrInst>(Val: IterInst->getNextNode());
3003	ASSERT_NE(DependBaseAddrGEP, nullptr);
3004	EXPECT_EQ(DependBaseAddrGEP->getPointerOperand(), AllocInst);
3005	EXPECT_EQ(DependBaseAddrGEP->getNumIndices(), (unsigned)2);
3006	auto *FirstIdx = dyn_cast<ConstantInt>(Val: DependBaseAddrGEP->getOperand(i_nocapture: 1));
3007	auto *SecondIdx = dyn_cast<ConstantInt>(Val: DependBaseAddrGEP->getOperand(i_nocapture: 2));
3008	ASSERT_NE(FirstIdx, nullptr);
3009	ASSERT_NE(SecondIdx, nullptr);
3010	EXPECT_EQ(FirstIdx->getValue(), 0);
3011	EXPECT_EQ(SecondIdx->getValue(), 0);
3012
3013	CallInst *GTID = dyn_cast<CallInst>(Val: DependBaseAddrGEP->getNextNode());
3014	ASSERT_NE(GTID, nullptr);
3015	EXPECT_EQ(GTID->arg_size(), 1U);
3016	EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
3017	EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
3018	EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
3019
3020	CallInst *Depend = dyn_cast<CallInst>(Val: GTID->getNextNode());
3021	ASSERT_NE(Depend, nullptr);
3022	EXPECT_EQ(Depend->arg_size(), 3U);
3023	EXPECT_EQ(Depend->getCalledFunction()->getName(), "__kmpc_doacross_post");
3024	EXPECT_TRUE(isa<GlobalVariable>(Depend->getArgOperand(0)));
3025	EXPECT_EQ(Depend->getArgOperand(1), GTID);
3026	EXPECT_EQ(Depend->getArgOperand(2), DependBaseAddrGEP);
3027	}
3028
3029	TEST_F(OpenMPIRBuilderTest, OrderedDirectiveDependSink) {
3030	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
3031	OpenMPIRBuilder OMPBuilder(*M);
3032	OMPBuilder.initialize();
3033	F->setName("func");
3034	IRBuilder<> Builder(BB);
3035	LLVMContext &Ctx = M->getContext();
3036
3037	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3038
3039	InsertPointTy AllocaIP(&F->getEntryBlock(),
3040	F->getEntryBlock().getFirstInsertionPt());
3041
3042	unsigned NumLoops = 2;
3043	SmallVector<Value *, 2> StoreValues;
3044	Type *LCTy = Type::getInt64Ty(C&: Ctx);
3045	StoreValues.emplace_back(Args: ConstantInt::get(Ty: LCTy, V: 1));
3046	StoreValues.emplace_back(Args: ConstantInt::get(Ty: LCTy, V: 2));
3047
3048	// Test for "#omp ordered depend(sink: vec)"
3049	Builder.restoreIP(IP: OMPBuilder.createOrderedDepend(Loc: Builder, AllocaIP, NumLoops,
3050	StoreValues, Name: ".cnt.addr",
3051	/*IsDependSource=*/false));
3052
3053	Builder.CreateRetVoid();
3054	OMPBuilder.finalize();
3055	EXPECT_FALSE(verifyModule(*M, &errs()));
3056
3057	AllocaInst *AllocInst = dyn_cast<AllocaInst>(Val: &BB->front());
3058	ASSERT_NE(AllocInst, nullptr);
3059	ArrayType *ArrType = dyn_cast<ArrayType>(Val: AllocInst->getAllocatedType());
3060	EXPECT_EQ(ArrType->getNumElements(), NumLoops);
3061	EXPECT_TRUE(
3062	AllocInst->getAllocatedType()->getArrayElementType()->isIntegerTy(64));
3063
3064	Instruction *IterInst = dyn_cast<Instruction>(Val: AllocInst);
3065	for (unsigned Iter = 0; Iter < NumLoops; Iter++) {
3066	GetElementPtrInst *DependAddrGEPIter =
3067	dyn_cast<GetElementPtrInst>(Val: IterInst->getNextNode());
3068	ASSERT_NE(DependAddrGEPIter, nullptr);
3069	EXPECT_EQ(DependAddrGEPIter->getPointerOperand(), AllocInst);
3070	EXPECT_EQ(DependAddrGEPIter->getNumIndices(), (unsigned)2);
3071	auto *FirstIdx = dyn_cast<ConstantInt>(Val: DependAddrGEPIter->getOperand(i_nocapture: 1));
3072	auto *SecondIdx = dyn_cast<ConstantInt>(Val: DependAddrGEPIter->getOperand(i_nocapture: 2));
3073	ASSERT_NE(FirstIdx, nullptr);
3074	ASSERT_NE(SecondIdx, nullptr);
3075	EXPECT_EQ(FirstIdx->getValue(), 0);
3076	EXPECT_EQ(SecondIdx->getValue(), Iter);
3077	StoreInst *StoreValue =
3078	dyn_cast<StoreInst>(Val: DependAddrGEPIter->getNextNode());
3079	ASSERT_NE(StoreValue, nullptr);
3080	EXPECT_EQ(StoreValue->getValueOperand(), StoreValues[Iter]);
3081	EXPECT_EQ(StoreValue->getPointerOperand(), DependAddrGEPIter);
3082	EXPECT_EQ(StoreValue->getAlign(), Align(8));
3083	IterInst = dyn_cast<Instruction>(Val: StoreValue);
3084	}
3085
3086	GetElementPtrInst *DependBaseAddrGEP =
3087	dyn_cast<GetElementPtrInst>(Val: IterInst->getNextNode());
3088	ASSERT_NE(DependBaseAddrGEP, nullptr);
3089	EXPECT_EQ(DependBaseAddrGEP->getPointerOperand(), AllocInst);
3090	EXPECT_EQ(DependBaseAddrGEP->getNumIndices(), (unsigned)2);
3091	auto *FirstIdx = dyn_cast<ConstantInt>(Val: DependBaseAddrGEP->getOperand(i_nocapture: 1));
3092	auto *SecondIdx = dyn_cast<ConstantInt>(Val: DependBaseAddrGEP->getOperand(i_nocapture: 2));
3093	ASSERT_NE(FirstIdx, nullptr);
3094	ASSERT_NE(SecondIdx, nullptr);
3095	EXPECT_EQ(FirstIdx->getValue(), 0);
3096	EXPECT_EQ(SecondIdx->getValue(), 0);
3097
3098	CallInst *GTID = dyn_cast<CallInst>(Val: DependBaseAddrGEP->getNextNode());
3099	ASSERT_NE(GTID, nullptr);
3100	EXPECT_EQ(GTID->arg_size(), 1U);
3101	EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
3102	EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
3103	EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
3104
3105	CallInst *Depend = dyn_cast<CallInst>(Val: GTID->getNextNode());
3106	ASSERT_NE(Depend, nullptr);
3107	EXPECT_EQ(Depend->arg_size(), 3U);
3108	EXPECT_EQ(Depend->getCalledFunction()->getName(), "__kmpc_doacross_wait");
3109	EXPECT_TRUE(isa<GlobalVariable>(Depend->getArgOperand(0)));
3110	EXPECT_EQ(Depend->getArgOperand(1), GTID);
3111	EXPECT_EQ(Depend->getArgOperand(2), DependBaseAddrGEP);
3112	}
3113
3114	TEST_F(OpenMPIRBuilderTest, OrderedDirectiveThreads) {
3115	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
3116	OpenMPIRBuilder OMPBuilder(*M);
3117	OMPBuilder.initialize();
3118	F->setName("func");
3119	IRBuilder<> Builder(BB);
3120
3121	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3122
3123	AllocaInst *PrivAI =
3124	Builder.CreateAlloca(Ty: F->arg_begin()->getType(), ArraySize: nullptr, Name: "priv.inst");
3125
3126	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
3127	llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
3128	llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
3129	EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
3130
3131	Builder.restoreIP(IP: CodeGenIP);
3132	Builder.CreateStore(Val: F->arg_begin(), Ptr: PrivAI);
3133	Value *PrivLoad =
3134	Builder.CreateLoad(Ty: PrivAI->getAllocatedType(), Ptr: PrivAI, Name: "local.use");
3135	Builder.CreateICmpNE(LHS: F->arg_begin(), RHS: PrivLoad);
3136	};
3137
3138	auto FiniCB = [&](InsertPointTy IP) {
3139	BasicBlock *IPBB = IP.getBlock();
3140	EXPECT_NE(IPBB->end(), IP.getPoint());
3141	};
3142
3143	// Test for "#omp ordered [threads]"
3144	BasicBlock *EntryBB = Builder.GetInsertBlock();
3145	Builder.restoreIP(
3146	IP: OMPBuilder.createOrderedThreadsSimd(Loc: Builder, BodyGenCB, FiniCB, IsThreads: true));
3147
3148	Builder.CreateRetVoid();
3149	OMPBuilder.finalize();
3150	EXPECT_FALSE(verifyModule(*M, &errs()));
3151
3152	EXPECT_NE(EntryBB->getTerminator(), nullptr);
3153
3154	CallInst *OrderedEntryCI = nullptr;
3155	for (auto &EI : *EntryBB) {
3156	Instruction *Cur = &EI;
3157	if (isa<CallInst>(Val: Cur)) {
3158	OrderedEntryCI = cast<CallInst>(Val: Cur);
3159	if (OrderedEntryCI->getCalledFunction()->getName() == "__kmpc_ordered")
3160	break;
3161	OrderedEntryCI = nullptr;
3162	}
3163	}
3164	EXPECT_NE(OrderedEntryCI, nullptr);
3165	EXPECT_EQ(OrderedEntryCI->arg_size(), 2U);
3166	EXPECT_EQ(OrderedEntryCI->getCalledFunction()->getName(), "__kmpc_ordered");
3167	EXPECT_TRUE(isa<GlobalVariable>(OrderedEntryCI->getArgOperand(0)));
3168
3169	CallInst *OrderedEndCI = nullptr;
3170	for (auto &FI : *EntryBB) {
3171	Instruction *Cur = &FI;
3172	if (isa<CallInst>(Val: Cur)) {
3173	OrderedEndCI = cast<CallInst>(Val: Cur);
3174	if (OrderedEndCI->getCalledFunction()->getName() == "__kmpc_end_ordered")
3175	break;
3176	OrderedEndCI = nullptr;
3177	}
3178	}
3179	EXPECT_NE(OrderedEndCI, nullptr);
3180	EXPECT_EQ(OrderedEndCI->arg_size(), 2U);
3181	EXPECT_TRUE(isa<GlobalVariable>(OrderedEndCI->getArgOperand(0)));
3182	EXPECT_EQ(OrderedEndCI->getArgOperand(1), OrderedEntryCI->getArgOperand(1));
3183	}
3184
3185	TEST_F(OpenMPIRBuilderTest, OrderedDirectiveSimd) {
3186	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
3187	OpenMPIRBuilder OMPBuilder(*M);
3188	OMPBuilder.initialize();
3189	F->setName("func");
3190	IRBuilder<> Builder(BB);
3191
3192	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3193
3194	AllocaInst *PrivAI =
3195	Builder.CreateAlloca(Ty: F->arg_begin()->getType(), ArraySize: nullptr, Name: "priv.inst");
3196
3197	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
3198	llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
3199	llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
3200	EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
3201
3202	Builder.restoreIP(IP: CodeGenIP);
3203	Builder.CreateStore(Val: F->arg_begin(), Ptr: PrivAI);
3204	Value *PrivLoad =
3205	Builder.CreateLoad(Ty: PrivAI->getAllocatedType(), Ptr: PrivAI, Name: "local.use");
3206	Builder.CreateICmpNE(LHS: F->arg_begin(), RHS: PrivLoad);
3207	};
3208
3209	auto FiniCB = [&](InsertPointTy IP) {
3210	BasicBlock *IPBB = IP.getBlock();
3211	EXPECT_NE(IPBB->end(), IP.getPoint());
3212	};
3213
3214	// Test for "#omp ordered simd"
3215	BasicBlock *EntryBB = Builder.GetInsertBlock();
3216	Builder.restoreIP(
3217	IP: OMPBuilder.createOrderedThreadsSimd(Loc: Builder, BodyGenCB, FiniCB, IsThreads: false));
3218
3219	Builder.CreateRetVoid();
3220	OMPBuilder.finalize();
3221	EXPECT_FALSE(verifyModule(*M, &errs()));
3222
3223	EXPECT_NE(EntryBB->getTerminator(), nullptr);
3224
3225	CallInst *OrderedEntryCI = nullptr;
3226	for (auto &EI : *EntryBB) {
3227	Instruction *Cur = &EI;
3228	if (isa<CallInst>(Val: Cur)) {
3229	OrderedEntryCI = cast<CallInst>(Val: Cur);
3230	if (OrderedEntryCI->getCalledFunction()->getName() == "__kmpc_ordered")
3231	break;
3232	OrderedEntryCI = nullptr;
3233	}
3234	}
3235	EXPECT_EQ(OrderedEntryCI, nullptr);
3236
3237	CallInst *OrderedEndCI = nullptr;
3238	for (auto &FI : *EntryBB) {
3239	Instruction *Cur = &FI;
3240	if (isa<CallInst>(Val: Cur)) {
3241	OrderedEndCI = cast<CallInst>(Val: Cur);
3242	if (OrderedEndCI->getCalledFunction()->getName() == "__kmpc_end_ordered")
3243	break;
3244	OrderedEndCI = nullptr;
3245	}
3246	}
3247	EXPECT_EQ(OrderedEndCI, nullptr);
3248	}
3249
3250	TEST_F(OpenMPIRBuilderTest, CopyinBlocks) {
3251	OpenMPIRBuilder OMPBuilder(*M);
3252	OMPBuilder.initialize();
3253	F->setName("func");
3254	IRBuilder<> Builder(BB);
3255
3256	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3257
3258	IntegerType *Int32 = Type::getInt32Ty(C&: M->getContext());
3259	AllocaInst *MasterAddress = Builder.CreateAlloca(Ty: Builder.getPtrTy());
3260	AllocaInst *PrivAddress = Builder.CreateAlloca(Ty: Builder.getPtrTy());
3261
3262	BasicBlock *EntryBB = BB;
3263
3264	OMPBuilder.createCopyinClauseBlocks(IP: Builder.saveIP(), MasterAddr: MasterAddress,
3265	PrivateAddr: PrivAddress, IntPtrTy: Int32, /*BranchtoEnd*/ true);
3266
3267	BranchInst *EntryBr = dyn_cast_or_null<BranchInst>(Val: EntryBB->getTerminator());
3268
3269	EXPECT_NE(EntryBr, nullptr);
3270	EXPECT_TRUE(EntryBr->isConditional());
3271
3272	BasicBlock *NotMasterBB = EntryBr->getSuccessor(i: 0);
3273	BasicBlock *CopyinEnd = EntryBr->getSuccessor(i: 1);
3274	CmpInst *CMP = dyn_cast_or_null<CmpInst>(Val: EntryBr->getCondition());
3275
3276	EXPECT_NE(CMP, nullptr);
3277	EXPECT_NE(NotMasterBB, nullptr);
3278	EXPECT_NE(CopyinEnd, nullptr);
3279
3280	BranchInst *NotMasterBr =
3281	dyn_cast_or_null<BranchInst>(Val: NotMasterBB->getTerminator());
3282	EXPECT_NE(NotMasterBr, nullptr);
3283	EXPECT_FALSE(NotMasterBr->isConditional());
3284	EXPECT_EQ(CopyinEnd, NotMasterBr->getSuccessor(0));
3285	}
3286
3287	TEST_F(OpenMPIRBuilderTest, SingleDirective) {
3288	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
3289	OpenMPIRBuilder OMPBuilder(*M);
3290	OMPBuilder.initialize();
3291	F->setName("func");
3292	IRBuilder<> Builder(BB);
3293
3294	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3295
3296	AllocaInst *PrivAI = nullptr;
3297
3298	BasicBlock *EntryBB = nullptr;
3299	BasicBlock *ThenBB = nullptr;
3300
3301	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
3302	if (AllocaIP.isSet())
3303	Builder.restoreIP(IP: AllocaIP);
3304	else
3305	Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
3306	PrivAI = Builder.CreateAlloca(Ty: F->arg_begin()->getType());
3307	Builder.CreateStore(Val: F->arg_begin(), Ptr: PrivAI);
3308
3309	llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
3310	llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
3311	EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
3312
3313	Builder.restoreIP(IP: CodeGenIP);
3314
3315	// collect some info for checks later
3316	ThenBB = Builder.GetInsertBlock();
3317	EntryBB = ThenBB->getUniquePredecessor();
3318
3319	// simple instructions for body
3320	Value *PrivLoad =
3321	Builder.CreateLoad(Ty: PrivAI->getAllocatedType(), Ptr: PrivAI, Name: "local.use");
3322	Builder.CreateICmpNE(LHS: F->arg_begin(), RHS: PrivLoad);
3323	};
3324
3325	auto FiniCB = [&](InsertPointTy IP) {
3326	BasicBlock *IPBB = IP.getBlock();
3327	EXPECT_NE(IPBB->end(), IP.getPoint());
3328	};
3329
3330	Builder.restoreIP(
3331	IP: OMPBuilder.createSingle(Loc: Builder, BodyGenCB, FiniCB, /*IsNowait*/ false));
3332	Value *EntryBBTI = EntryBB->getTerminator();
3333	EXPECT_NE(EntryBBTI, nullptr);
3334	EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
3335	BranchInst *EntryBr = cast<BranchInst>(Val: EntryBB->getTerminator());
3336	EXPECT_TRUE(EntryBr->isConditional());
3337	EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
3338	BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
3339	EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
3340
3341	CmpInst *CondInst = cast<CmpInst>(Val: EntryBr->getCondition());
3342	EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
3343
3344	CallInst *SingleEntryCI = cast<CallInst>(Val: CondInst->getOperand(i_nocapture: 0));
3345	EXPECT_EQ(SingleEntryCI->arg_size(), 2U);
3346	EXPECT_EQ(SingleEntryCI->getCalledFunction()->getName(), "__kmpc_single");
3347	EXPECT_TRUE(isa<GlobalVariable>(SingleEntryCI->getArgOperand(0)));
3348
3349	CallInst *SingleEndCI = nullptr;
3350	for (auto &FI : *ThenBB) {
3351	Instruction *cur = &FI;
3352	if (isa<CallInst>(Val: cur)) {
3353	SingleEndCI = cast<CallInst>(Val: cur);
3354	if (SingleEndCI->getCalledFunction()->getName() == "__kmpc_end_single")
3355	break;
3356	SingleEndCI = nullptr;
3357	}
3358	}
3359	EXPECT_NE(SingleEndCI, nullptr);
3360	EXPECT_EQ(SingleEndCI->arg_size(), 2U);
3361	EXPECT_TRUE(isa<GlobalVariable>(SingleEndCI->getArgOperand(0)));
3362	EXPECT_EQ(SingleEndCI->getArgOperand(1), SingleEntryCI->getArgOperand(1));
3363
3364	bool FoundBarrier = false;
3365	for (auto &FI : *ExitBB) {
3366	Instruction *cur = &FI;
3367	if (auto CI = dyn_cast<CallInst>(Val: cur)) {
3368	if (CI->getCalledFunction()->getName() == "__kmpc_barrier") {
3369	FoundBarrier = true;
3370	break;
3371	}
3372	}
3373	}
3374	EXPECT_TRUE(FoundBarrier);
3375	}
3376
3377	TEST_F(OpenMPIRBuilderTest, SingleDirectiveNowait) {
3378	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
3379	OpenMPIRBuilder OMPBuilder(*M);
3380	OMPBuilder.initialize();
3381	F->setName("func");
3382	IRBuilder<> Builder(BB);
3383
3384	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3385
3386	AllocaInst *PrivAI = nullptr;
3387
3388	BasicBlock *EntryBB = nullptr;
3389	BasicBlock *ThenBB = nullptr;
3390
3391	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
3392	if (AllocaIP.isSet())
3393	Builder.restoreIP(IP: AllocaIP);
3394	else
3395	Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
3396	PrivAI = Builder.CreateAlloca(Ty: F->arg_begin()->getType());
3397	Builder.CreateStore(Val: F->arg_begin(), Ptr: PrivAI);
3398
3399	llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
3400	llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
3401	EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
3402
3403	Builder.restoreIP(IP: CodeGenIP);
3404
3405	// collect some info for checks later
3406	ThenBB = Builder.GetInsertBlock();
3407	EntryBB = ThenBB->getUniquePredecessor();
3408
3409	// simple instructions for body
3410	Value *PrivLoad =
3411	Builder.CreateLoad(Ty: PrivAI->getAllocatedType(), Ptr: PrivAI, Name: "local.use");
3412	Builder.CreateICmpNE(LHS: F->arg_begin(), RHS: PrivLoad);
3413	};
3414
3415	auto FiniCB = [&](InsertPointTy IP) {
3416	BasicBlock *IPBB = IP.getBlock();
3417	EXPECT_NE(IPBB->end(), IP.getPoint());
3418	};
3419
3420	Builder.restoreIP(
3421	IP: OMPBuilder.createSingle(Loc: Builder, BodyGenCB, FiniCB, /*IsNowait*/ true));
3422	Value *EntryBBTI = EntryBB->getTerminator();
3423	EXPECT_NE(EntryBBTI, nullptr);
3424	EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
3425	BranchInst *EntryBr = cast<BranchInst>(Val: EntryBB->getTerminator());
3426	EXPECT_TRUE(EntryBr->isConditional());
3427	EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
3428	BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
3429	EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
3430
3431	CmpInst *CondInst = cast<CmpInst>(Val: EntryBr->getCondition());
3432	EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
3433
3434	CallInst *SingleEntryCI = cast<CallInst>(Val: CondInst->getOperand(i_nocapture: 0));
3435	EXPECT_EQ(SingleEntryCI->arg_size(), 2U);
3436	EXPECT_EQ(SingleEntryCI->getCalledFunction()->getName(), "__kmpc_single");
3437	EXPECT_TRUE(isa<GlobalVariable>(SingleEntryCI->getArgOperand(0)));
3438
3439	CallInst *SingleEndCI = nullptr;
3440	for (auto &FI : *ThenBB) {
3441	Instruction *cur = &FI;
3442	if (isa<CallInst>(Val: cur)) {
3443	SingleEndCI = cast<CallInst>(Val: cur);
3444	if (SingleEndCI->getCalledFunction()->getName() == "__kmpc_end_single")
3445	break;
3446	SingleEndCI = nullptr;
3447	}
3448	}
3449	EXPECT_NE(SingleEndCI, nullptr);
3450	EXPECT_EQ(SingleEndCI->arg_size(), 2U);
3451	EXPECT_TRUE(isa<GlobalVariable>(SingleEndCI->getArgOperand(0)));
3452	EXPECT_EQ(SingleEndCI->getArgOperand(1), SingleEntryCI->getArgOperand(1));
3453
3454	CallInst *ExitBarrier = nullptr;
3455	for (auto &FI : *ExitBB) {
3456	Instruction *cur = &FI;
3457	if (auto CI = dyn_cast<CallInst>(Val: cur)) {
3458	if (CI->getCalledFunction()->getName() == "__kmpc_barrier") {
3459	ExitBarrier = CI;
3460	break;
3461	}
3462	}
3463	}
3464	EXPECT_EQ(ExitBarrier, nullptr);
3465	}
3466
3467	// Helper class to check each instruction of a BB.
3468	class BBInstIter {
3469	BasicBlock *BB;
3470	BasicBlock::iterator BBI;
3471
3472	public:
3473	BBInstIter(BasicBlock *BB) : BB(BB), BBI(BB->begin()) {}
3474
3475	bool hasNext() const { return BBI != BB->end(); }
3476
3477	template <typename InstTy> InstTy *next() {
3478	if (!hasNext())
3479	return nullptr;
3480	Instruction *Cur = &*BBI++;
3481	if (!isa<InstTy>(Cur))
3482	return nullptr;
3483	return cast<InstTy>(Cur);
3484	}
3485	};
3486
3487	TEST_F(OpenMPIRBuilderTest, SingleDirectiveCopyPrivate) {
3488	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
3489	OpenMPIRBuilder OMPBuilder(*M);
3490	OMPBuilder.initialize();
3491	F->setName("func");
3492	IRBuilder<> Builder(BB);
3493
3494	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3495
3496	AllocaInst *PrivAI = nullptr;
3497
3498	BasicBlock *EntryBB = nullptr;
3499	BasicBlock *ThenBB = nullptr;
3500
3501	Value *CPVar = Builder.CreateAlloca(Ty: F->arg_begin()->getType());
3502	Builder.CreateStore(Val: F->arg_begin(), Ptr: CPVar);
3503
3504	FunctionType *CopyFuncTy = FunctionType::get(
3505	Result: Builder.getVoidTy(), Params: {Builder.getPtrTy(), Builder.getPtrTy()}, isVarArg: false);
3506	Function *CopyFunc =
3507	Function::Create(Ty: CopyFuncTy, Linkage: Function::PrivateLinkage, N: "copy_var", M&: *M);
3508
3509	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
3510	if (AllocaIP.isSet())
3511	Builder.restoreIP(IP: AllocaIP);
3512	else
3513	Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
3514	PrivAI = Builder.CreateAlloca(Ty: F->arg_begin()->getType());
3515	Builder.CreateStore(Val: F->arg_begin(), Ptr: PrivAI);
3516
3517	llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
3518	llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
3519	EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
3520
3521	Builder.restoreIP(IP: CodeGenIP);
3522
3523	// collect some info for checks later
3524	ThenBB = Builder.GetInsertBlock();
3525	EntryBB = ThenBB->getUniquePredecessor();
3526
3527	// simple instructions for body
3528	Value *PrivLoad =
3529	Builder.CreateLoad(Ty: PrivAI->getAllocatedType(), Ptr: PrivAI, Name: "local.use");
3530	Builder.CreateICmpNE(LHS: F->arg_begin(), RHS: PrivLoad);
3531	};
3532
3533	auto FiniCB = [&](InsertPointTy IP) {
3534	BasicBlock *IPBB = IP.getBlock();
3535	// IP must be before the unconditional branch to ExitBB
3536	EXPECT_NE(IPBB->end(), IP.getPoint());
3537	};
3538
3539	Builder.restoreIP(IP: OMPBuilder.createSingle(Loc: Builder, BodyGenCB, FiniCB,
3540	/*IsNowait*/ false, CPVars: {CPVar},
3541	CPFuncs: {CopyFunc}));
3542	Value *EntryBBTI = EntryBB->getTerminator();
3543	EXPECT_NE(EntryBBTI, nullptr);
3544	EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
3545	BranchInst *EntryBr = cast<BranchInst>(Val: EntryBB->getTerminator());
3546	EXPECT_TRUE(EntryBr->isConditional());
3547	EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
3548	BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
3549	EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
3550
3551	CmpInst *CondInst = cast<CmpInst>(Val: EntryBr->getCondition());
3552	EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
3553
3554	CallInst *SingleEntryCI = cast<CallInst>(Val: CondInst->getOperand(i_nocapture: 0));
3555	EXPECT_EQ(SingleEntryCI->arg_size(), 2U);
3556	EXPECT_EQ(SingleEntryCI->getCalledFunction()->getName(), "__kmpc_single");
3557	EXPECT_TRUE(isa<GlobalVariable>(SingleEntryCI->getArgOperand(0)));
3558
3559	// check ThenBB
3560	BBInstIter ThenBBI(ThenBB);
3561	// load PrivAI
3562	auto *PrivLI = ThenBBI.next<LoadInst>();
3563	EXPECT_NE(PrivLI, nullptr);
3564	EXPECT_EQ(PrivLI->getPointerOperand(), PrivAI);
3565	// icmp
3566	EXPECT_TRUE(ThenBBI.next<ICmpInst>());
3567	// store 1, DidIt
3568	auto *DidItSI = ThenBBI.next<StoreInst>();
3569	EXPECT_NE(DidItSI, nullptr);
3570	EXPECT_EQ(DidItSI->getValueOperand(),
3571	ConstantInt::get(Type::getInt32Ty(Ctx), 1));
3572	Value *DidIt = DidItSI->getPointerOperand();
3573	// call __kmpc_end_single
3574	auto *SingleEndCI = ThenBBI.next<CallInst>();
3575	EXPECT_NE(SingleEndCI, nullptr);
3576	EXPECT_EQ(SingleEndCI->getCalledFunction()->getName(), "__kmpc_end_single");
3577	EXPECT_EQ(SingleEndCI->arg_size(), 2U);
3578	EXPECT_TRUE(isa<GlobalVariable>(SingleEndCI->getArgOperand(0)));
3579	EXPECT_EQ(SingleEndCI->getArgOperand(1), SingleEntryCI->getArgOperand(1));
3580	// br ExitBB
3581	auto *ExitBBBI = ThenBBI.next<BranchInst>();
3582	EXPECT_NE(ExitBBBI, nullptr);
3583	EXPECT_TRUE(ExitBBBI->isUnconditional());
3584	EXPECT_EQ(ExitBBBI->getOperand(0), ExitBB);
3585	EXPECT_FALSE(ThenBBI.hasNext());
3586
3587	// check ExitBB
3588	BBInstIter ExitBBI(ExitBB);
3589	// call __kmpc_global_thread_num
3590	auto *ThreadNumCI = ExitBBI.next<CallInst>();
3591	EXPECT_NE(ThreadNumCI, nullptr);
3592	EXPECT_EQ(ThreadNumCI->getCalledFunction()->getName(),
3593	"__kmpc_global_thread_num");
3594	// load DidIt
3595	auto *DidItLI = ExitBBI.next<LoadInst>();
3596	EXPECT_NE(DidItLI, nullptr);
3597	EXPECT_EQ(DidItLI->getPointerOperand(), DidIt);
3598	// call __kmpc_copyprivate
3599	auto *CopyPrivateCI = ExitBBI.next<CallInst>();
3600	EXPECT_NE(CopyPrivateCI, nullptr);
3601	EXPECT_EQ(CopyPrivateCI->arg_size(), 6U);
3602	EXPECT_TRUE(isa<AllocaInst>(CopyPrivateCI->getArgOperand(3)));
3603	EXPECT_EQ(CopyPrivateCI->getArgOperand(3), CPVar);
3604	EXPECT_TRUE(isa<Function>(CopyPrivateCI->getArgOperand(4)));
3605	EXPECT_EQ(CopyPrivateCI->getArgOperand(4), CopyFunc);
3606	EXPECT_TRUE(isa<LoadInst>(CopyPrivateCI->getArgOperand(5)));
3607	DidItLI = cast<LoadInst>(Val: CopyPrivateCI->getArgOperand(i: 5));
3608	EXPECT_EQ(DidItLI->getOperand(0), DidIt);
3609	EXPECT_FALSE(ExitBBI.hasNext());
3610	}
3611
3612	TEST_F(OpenMPIRBuilderTest, OMPAtomicReadFlt) {
3613	OpenMPIRBuilder OMPBuilder(*M);
3614	OMPBuilder.initialize();
3615	F->setName("func");
3616	IRBuilder<> Builder(BB);
3617
3618	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3619
3620	Type *Float32 = Type::getFloatTy(C&: M->getContext());
3621	AllocaInst *XVal = Builder.CreateAlloca(Ty: Float32);
3622	XVal->setName("AtomicVar");
3623	AllocaInst *VVal = Builder.CreateAlloca(Ty: Float32);
3624	VVal->setName("AtomicRead");
3625	AtomicOrdering AO = AtomicOrdering::Monotonic;
3626	OpenMPIRBuilder::AtomicOpValue X = {.Var: XVal, .ElemTy: Float32, .IsSigned: false, .IsVolatile: false};
3627	OpenMPIRBuilder::AtomicOpValue V = {.Var: VVal, .ElemTy: Float32, .IsSigned: false, .IsVolatile: false};
3628
3629	Builder.restoreIP(IP: OMPBuilder.createAtomicRead(Loc, X, V, AO));
3630
3631	IntegerType *IntCastTy =
3632	IntegerType::get(C&: M->getContext(), NumBits: Float32->getScalarSizeInBits());
3633
3634	LoadInst *AtomicLoad = cast<LoadInst>(Val: VVal->getNextNode());
3635	EXPECT_TRUE(AtomicLoad->isAtomic());
3636	EXPECT_EQ(AtomicLoad->getPointerOperand(), XVal);
3637
3638	BitCastInst *CastToFlt = cast<BitCastInst>(Val: AtomicLoad->getNextNode());
3639	EXPECT_EQ(CastToFlt->getSrcTy(), IntCastTy);
3640	EXPECT_EQ(CastToFlt->getDestTy(), Float32);
3641	EXPECT_EQ(CastToFlt->getOperand(0), AtomicLoad);
3642
3643	StoreInst *StoreofAtomic = cast<StoreInst>(Val: CastToFlt->getNextNode());
3644	EXPECT_EQ(StoreofAtomic->getValueOperand(), CastToFlt);
3645	EXPECT_EQ(StoreofAtomic->getPointerOperand(), VVal);
3646
3647	Builder.CreateRetVoid();
3648	OMPBuilder.finalize();
3649	EXPECT_FALSE(verifyModule(*M, &errs()));
3650	}
3651
3652	TEST_F(OpenMPIRBuilderTest, OMPAtomicReadInt) {
3653	OpenMPIRBuilder OMPBuilder(*M);
3654	OMPBuilder.initialize();
3655	F->setName("func");
3656	IRBuilder<> Builder(BB);
3657
3658	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3659
3660	IntegerType *Int32 = Type::getInt32Ty(C&: M->getContext());
3661	AllocaInst *XVal = Builder.CreateAlloca(Ty: Int32);
3662	XVal->setName("AtomicVar");
3663	AllocaInst *VVal = Builder.CreateAlloca(Ty: Int32);
3664	VVal->setName("AtomicRead");
3665	AtomicOrdering AO = AtomicOrdering::Monotonic;
3666	OpenMPIRBuilder::AtomicOpValue X = {.Var: XVal, .ElemTy: Int32, .IsSigned: false, .IsVolatile: false};
3667	OpenMPIRBuilder::AtomicOpValue V = {.Var: VVal, .ElemTy: Int32, .IsSigned: false, .IsVolatile: false};
3668
3669	BasicBlock *EntryBB = BB;
3670
3671	Builder.restoreIP(IP: OMPBuilder.createAtomicRead(Loc, X, V, AO));
3672	LoadInst *AtomicLoad = nullptr;
3673	StoreInst *StoreofAtomic = nullptr;
3674
3675	for (Instruction &Cur : *EntryBB) {
3676	if (isa<LoadInst>(Val: Cur)) {
3677	AtomicLoad = cast<LoadInst>(Val: &Cur);
3678	if (AtomicLoad->getPointerOperand() == XVal)
3679	continue;
3680	AtomicLoad = nullptr;
3681	} else if (isa<StoreInst>(Val: Cur)) {
3682	StoreofAtomic = cast<StoreInst>(Val: &Cur);
3683	if (StoreofAtomic->getPointerOperand() == VVal)
3684	continue;
3685	StoreofAtomic = nullptr;
3686	}
3687	}
3688
3689	EXPECT_NE(AtomicLoad, nullptr);
3690	EXPECT_TRUE(AtomicLoad->isAtomic());
3691
3692	EXPECT_NE(StoreofAtomic, nullptr);
3693	EXPECT_EQ(StoreofAtomic->getValueOperand(), AtomicLoad);
3694
3695	Builder.CreateRetVoid();
3696	OMPBuilder.finalize();
3697
3698	EXPECT_FALSE(verifyModule(*M, &errs()));
3699	}
3700
3701	TEST_F(OpenMPIRBuilderTest, OMPAtomicWriteFlt) {
3702	OpenMPIRBuilder OMPBuilder(*M);
3703	OMPBuilder.initialize();
3704	F->setName("func");
3705	IRBuilder<> Builder(BB);
3706
3707	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3708
3709	LLVMContext &Ctx = M->getContext();
3710	Type *Float32 = Type::getFloatTy(C&: Ctx);
3711	AllocaInst *XVal = Builder.CreateAlloca(Ty: Float32);
3712	XVal->setName("AtomicVar");
3713	OpenMPIRBuilder::AtomicOpValue X = {.Var: XVal, .ElemTy: Float32, .IsSigned: false, .IsVolatile: false};
3714	AtomicOrdering AO = AtomicOrdering::Monotonic;
3715	Constant *ValToWrite = ConstantFP::get(Ty: Float32, V: 1.0);
3716
3717	Builder.restoreIP(IP: OMPBuilder.createAtomicWrite(Loc, X, Expr: ValToWrite, AO));
3718
3719	IntegerType *IntCastTy =
3720	IntegerType::get(C&: M->getContext(), NumBits: Float32->getScalarSizeInBits());
3721
3722	Value *ExprCast = Builder.CreateBitCast(V: ValToWrite, DestTy: IntCastTy);
3723
3724	StoreInst *StoreofAtomic = cast<StoreInst>(Val: XVal->getNextNode());
3725	EXPECT_EQ(StoreofAtomic->getValueOperand(), ExprCast);
3726	EXPECT_EQ(StoreofAtomic->getPointerOperand(), XVal);
3727	EXPECT_TRUE(StoreofAtomic->isAtomic());
3728
3729	Builder.CreateRetVoid();
3730	OMPBuilder.finalize();
3731	EXPECT_FALSE(verifyModule(*M, &errs()));
3732	}
3733
3734	TEST_F(OpenMPIRBuilderTest, OMPAtomicWriteInt) {
3735	OpenMPIRBuilder OMPBuilder(*M);
3736	OMPBuilder.initialize();
3737	F->setName("func");
3738	IRBuilder<> Builder(BB);
3739
3740	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3741
3742	LLVMContext &Ctx = M->getContext();
3743	IntegerType *Int32 = Type::getInt32Ty(C&: Ctx);
3744	AllocaInst *XVal = Builder.CreateAlloca(Ty: Int32);
3745	XVal->setName("AtomicVar");
3746	OpenMPIRBuilder::AtomicOpValue X = {.Var: XVal, .ElemTy: Int32, .IsSigned: false, .IsVolatile: false};
3747	AtomicOrdering AO = AtomicOrdering::Monotonic;
3748	ConstantInt *ValToWrite = ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 1U);
3749
3750	BasicBlock *EntryBB = BB;
3751
3752	Builder.restoreIP(IP: OMPBuilder.createAtomicWrite(Loc, X, Expr: ValToWrite, AO));
3753
3754	StoreInst *StoreofAtomic = nullptr;
3755
3756	for (Instruction &Cur : *EntryBB) {
3757	if (isa<StoreInst>(Val: Cur)) {
3758	StoreofAtomic = cast<StoreInst>(Val: &Cur);
3759	if (StoreofAtomic->getPointerOperand() == XVal)
3760	continue;
3761	StoreofAtomic = nullptr;
3762	}
3763	}
3764
3765	EXPECT_NE(StoreofAtomic, nullptr);
3766	EXPECT_TRUE(StoreofAtomic->isAtomic());
3767	EXPECT_EQ(StoreofAtomic->getValueOperand(), ValToWrite);
3768
3769	Builder.CreateRetVoid();
3770	OMPBuilder.finalize();
3771	EXPECT_FALSE(verifyModule(*M, &errs()));
3772	}
3773
3774	TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdate) {
3775	OpenMPIRBuilder OMPBuilder(*M);
3776	OMPBuilder.initialize();
3777	F->setName("func");
3778	IRBuilder<> Builder(BB);
3779
3780	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3781
3782	IntegerType *Int32 = Type::getInt32Ty(C&: M->getContext());
3783	AllocaInst *XVal = Builder.CreateAlloca(Ty: Int32);
3784	XVal->setName("AtomicVar");
3785	Builder.CreateStore(Val: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 0U), Ptr: XVal);
3786	OpenMPIRBuilder::AtomicOpValue X = {.Var: XVal, .ElemTy: Int32, .IsSigned: false, .IsVolatile: false};
3787	AtomicOrdering AO = AtomicOrdering::Monotonic;
3788	ConstantInt *ConstVal = ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 1U);
3789	Value *Expr = nullptr;
3790	AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::Sub;
3791	bool IsXLHSInRHSPart = false;
3792
3793	BasicBlock *EntryBB = BB;
3794	OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3795	EntryBB->getFirstInsertionPt());
3796	Value *Sub = nullptr;
3797
3798	auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {
3799	Sub = IRB.CreateSub(LHS: ConstVal, RHS: Atomic);
3800	return Sub;
3801	};
3802	Builder.restoreIP(IP: OMPBuilder.createAtomicUpdate(
3803	Loc: Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXBinopExpr: IsXLHSInRHSPart));
3804	BasicBlock *ContBB = EntryBB->getSingleSuccessor();
3805	BranchInst *ContTI = dyn_cast<BranchInst>(Val: ContBB->getTerminator());
3806	EXPECT_NE(ContTI, nullptr);
3807	BasicBlock *EndBB = ContTI->getSuccessor(i: 0);
3808	EXPECT_TRUE(ContTI->isConditional());
3809	EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
3810	EXPECT_NE(EndBB, nullptr);
3811
3812	PHINode *Phi = dyn_cast<PHINode>(Val: &ContBB->front());
3813	EXPECT_NE(Phi, nullptr);
3814	EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
3815	EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
3816	EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
3817
3818	EXPECT_EQ(Sub->getNumUses(), 1U);
3819	StoreInst *St = dyn_cast<StoreInst>(Val: Sub->user_back());
3820	AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(Val: St->getPointerOperand());
3821
3822	ExtractValueInst *ExVI1 =
3823	dyn_cast<ExtractValueInst>(Val: Phi->getIncomingValueForBlock(BB: ContBB));
3824	EXPECT_NE(ExVI1, nullptr);
3825	AtomicCmpXchgInst *CmpExchg =
3826	dyn_cast<AtomicCmpXchgInst>(Val: ExVI1->getAggregateOperand());
3827	EXPECT_NE(CmpExchg, nullptr);
3828	EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);
3829	EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
3830	EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
3831
3832	LoadInst *Ld = dyn_cast<LoadInst>(Val: CmpExchg->getNewValOperand());
3833	EXPECT_NE(Ld, nullptr);
3834	EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());
3835
3836	Builder.CreateRetVoid();
3837	OMPBuilder.finalize();
3838	EXPECT_FALSE(verifyModule(*M, &errs()));
3839	}
3840
3841	TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateFloat) {
3842	OpenMPIRBuilder OMPBuilder(*M);
3843	OMPBuilder.initialize();
3844	F->setName("func");
3845	IRBuilder<> Builder(BB);
3846
3847	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3848
3849	Type *FloatTy = Type::getFloatTy(C&: M->getContext());
3850	AllocaInst *XVal = Builder.CreateAlloca(Ty: FloatTy);
3851	XVal->setName("AtomicVar");
3852	Builder.CreateStore(Val: ConstantFP::get(Ty: Type::getFloatTy(C&: Ctx), V: 0.0), Ptr: XVal);
3853	OpenMPIRBuilder::AtomicOpValue X = {.Var: XVal, .ElemTy: FloatTy, .IsSigned: false, .IsVolatile: false};
3854	AtomicOrdering AO = AtomicOrdering::Monotonic;
3855	Constant *ConstVal = ConstantFP::get(Ty: Type::getFloatTy(C&: Ctx), V: 1.0);
3856	Value *Expr = nullptr;
3857	AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::FSub;
3858	bool IsXLHSInRHSPart = false;
3859
3860	BasicBlock *EntryBB = BB;
3861	OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3862	EntryBB->getFirstInsertionPt());
3863	Value *Sub = nullptr;
3864
3865	auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {
3866	Sub = IRB.CreateFSub(L: ConstVal, R: Atomic);
3867	return Sub;
3868	};
3869	Builder.restoreIP(IP: OMPBuilder.createAtomicUpdate(
3870	Loc: Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXBinopExpr: IsXLHSInRHSPart));
3871	BasicBlock *ContBB = EntryBB->getSingleSuccessor();
3872	BranchInst *ContTI = dyn_cast<BranchInst>(Val: ContBB->getTerminator());
3873	EXPECT_NE(ContTI, nullptr);
3874	BasicBlock *EndBB = ContTI->getSuccessor(i: 0);
3875	EXPECT_TRUE(ContTI->isConditional());
3876	EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
3877	EXPECT_NE(EndBB, nullptr);
3878
3879	PHINode *Phi = dyn_cast<PHINode>(Val: &ContBB->front());
3880	EXPECT_NE(Phi, nullptr);
3881	EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
3882	EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
3883	EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
3884
3885	EXPECT_EQ(Sub->getNumUses(), 1U);
3886	StoreInst *St = dyn_cast<StoreInst>(Val: Sub->user_back());
3887	AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(Val: St->getPointerOperand());
3888
3889	ExtractValueInst *ExVI1 =
3890	dyn_cast<ExtractValueInst>(Val: Phi->getIncomingValueForBlock(BB: ContBB));
3891	EXPECT_NE(ExVI1, nullptr);
3892	AtomicCmpXchgInst *CmpExchg =
3893	dyn_cast<AtomicCmpXchgInst>(Val: ExVI1->getAggregateOperand());
3894	EXPECT_NE(CmpExchg, nullptr);
3895	EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);
3896	EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
3897	EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
3898
3899	LoadInst *Ld = dyn_cast<LoadInst>(Val: CmpExchg->getNewValOperand());
3900	EXPECT_NE(Ld, nullptr);
3901	EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());
3902	Builder.CreateRetVoid();
3903	OMPBuilder.finalize();
3904	EXPECT_FALSE(verifyModule(*M, &errs()));
3905	}
3906
3907	TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateIntr) {
3908	OpenMPIRBuilder OMPBuilder(*M);
3909	OMPBuilder.initialize();
3910	F->setName("func");
3911	IRBuilder<> Builder(BB);
3912
3913	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3914
3915	Type *IntTy = Type::getInt32Ty(C&: M->getContext());
3916	AllocaInst *XVal = Builder.CreateAlloca(Ty: IntTy);
3917	XVal->setName("AtomicVar");
3918	Builder.CreateStore(Val: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 0), Ptr: XVal);
3919	OpenMPIRBuilder::AtomicOpValue X = {.Var: XVal, .ElemTy: IntTy, .IsSigned: false, .IsVolatile: false};
3920	AtomicOrdering AO = AtomicOrdering::Monotonic;
3921	Constant *ConstVal = ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 1);
3922	Value *Expr = ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 1);
3923	AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::UMax;
3924	bool IsXLHSInRHSPart = false;
3925
3926	BasicBlock *EntryBB = BB;
3927	OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3928	EntryBB->getFirstInsertionPt());
3929	Value *Sub = nullptr;
3930
3931	auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {
3932	Sub = IRB.CreateSub(LHS: ConstVal, RHS: Atomic);
3933	return Sub;
3934	};
3935	Builder.restoreIP(IP: OMPBuilder.createAtomicUpdate(
3936	Loc: Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXBinopExpr: IsXLHSInRHSPart));
3937	BasicBlock *ContBB = EntryBB->getSingleSuccessor();
3938	BranchInst *ContTI = dyn_cast<BranchInst>(Val: ContBB->getTerminator());
3939	EXPECT_NE(ContTI, nullptr);
3940	BasicBlock *EndBB = ContTI->getSuccessor(i: 0);
3941	EXPECT_TRUE(ContTI->isConditional());
3942	EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
3943	EXPECT_NE(EndBB, nullptr);
3944
3945	PHINode *Phi = dyn_cast<PHINode>(Val: &ContBB->front());
3946	EXPECT_NE(Phi, nullptr);
3947	EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
3948	EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
3949	EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
3950
3951	EXPECT_EQ(Sub->getNumUses(), 1U);
3952	StoreInst *St = dyn_cast<StoreInst>(Val: Sub->user_back());
3953	AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(Val: St->getPointerOperand());
3954
3955	ExtractValueInst *ExVI1 =
3956	dyn_cast<ExtractValueInst>(Val: Phi->getIncomingValueForBlock(BB: ContBB));
3957	EXPECT_NE(ExVI1, nullptr);
3958	AtomicCmpXchgInst *CmpExchg =
3959	dyn_cast<AtomicCmpXchgInst>(Val: ExVI1->getAggregateOperand());
3960	EXPECT_NE(CmpExchg, nullptr);
3961	EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);
3962	EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
3963	EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
3964
3965	LoadInst *Ld = dyn_cast<LoadInst>(Val: CmpExchg->getNewValOperand());
3966	EXPECT_NE(Ld, nullptr);
3967	EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());
3968
3969	Builder.CreateRetVoid();
3970	OMPBuilder.finalize();
3971	EXPECT_FALSE(verifyModule(*M, &errs()));
3972	}
3973
3974	TEST_F(OpenMPIRBuilderTest, OMPAtomicCapture) {
3975	OpenMPIRBuilder OMPBuilder(*M);
3976	OMPBuilder.initialize();
3977	F->setName("func");
3978	IRBuilder<> Builder(BB);
3979
3980	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3981
3982	LLVMContext &Ctx = M->getContext();
3983	IntegerType *Int32 = Type::getInt32Ty(C&: Ctx);
3984	AllocaInst *XVal = Builder.CreateAlloca(Ty: Int32);
3985	XVal->setName("AtomicVar");
3986	AllocaInst *VVal = Builder.CreateAlloca(Ty: Int32);
3987	VVal->setName("AtomicCapTar");
3988	StoreInst *Init =
3989	Builder.CreateStore(Val: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 0U), Ptr: XVal);
3990
3991	OpenMPIRBuilder::AtomicOpValue X = {.Var: XVal, .ElemTy: Int32, .IsSigned: false, .IsVolatile: false};
3992	OpenMPIRBuilder::AtomicOpValue V = {.Var: VVal, .ElemTy: Int32, .IsSigned: false, .IsVolatile: false};
3993	AtomicOrdering AO = AtomicOrdering::Monotonic;
3994	ConstantInt *Expr = ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 1U);
3995	AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::Add;
3996	bool IsXLHSInRHSPart = true;
3997	bool IsPostfixUpdate = true;
3998	bool UpdateExpr = true;
3999
4000	BasicBlock *EntryBB = BB;
4001	OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
4002	EntryBB->getFirstInsertionPt());
4003
4004	// integer update - not used
4005	auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) { return nullptr; };
4006
4007	Builder.restoreIP(IP: OMPBuilder.createAtomicCapture(
4008	Loc: Builder, AllocaIP, X, V, Expr, AO, RMWOp, UpdateOp, UpdateExpr,
4009	IsPostfixUpdate, IsXBinopExpr: IsXLHSInRHSPart));
4010	EXPECT_EQ(EntryBB->getParent()->size(), 1U);
4011	AtomicRMWInst *ARWM = dyn_cast<AtomicRMWInst>(Val: Init->getNextNode());
4012	EXPECT_NE(ARWM, nullptr);
4013	EXPECT_EQ(ARWM->getPointerOperand(), XVal);
4014	EXPECT_EQ(ARWM->getOperation(), RMWOp);
4015	StoreInst *St = dyn_cast<StoreInst>(Val: ARWM->user_back());
4016	EXPECT_NE(St, nullptr);
4017	EXPECT_EQ(St->getPointerOperand(), VVal);
4018
4019	Builder.CreateRetVoid();
4020	OMPBuilder.finalize();
4021	EXPECT_FALSE(verifyModule(*M, &errs()));
4022	}
4023
4024	TEST_F(OpenMPIRBuilderTest, OMPAtomicCompare) {
4025	OpenMPIRBuilder OMPBuilder(*M);
4026	OMPBuilder.initialize();
4027	F->setName("func");
4028	IRBuilder<> Builder(BB);
4029
4030	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4031
4032	LLVMContext &Ctx = M->getContext();
4033	IntegerType *Int32 = Type::getInt32Ty(C&: Ctx);
4034	AllocaInst *XVal = Builder.CreateAlloca(Ty: Int32);
4035	XVal->setName("x");
4036	StoreInst *Init =
4037	Builder.CreateStore(Val: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 0U), Ptr: XVal);
4038
4039	OpenMPIRBuilder::AtomicOpValue XSigned = {.Var: XVal, .ElemTy: Int32, .IsSigned: true, .IsVolatile: false};
4040	OpenMPIRBuilder::AtomicOpValue XUnsigned = {.Var: XVal, .ElemTy: Int32, .IsSigned: false, .IsVolatile: false};
4041	// V and R are not used in atomic compare
4042	OpenMPIRBuilder::AtomicOpValue V = {.Var: nullptr, .ElemTy: nullptr, .IsSigned: false, .IsVolatile: false};
4043	OpenMPIRBuilder::AtomicOpValue R = {.Var: nullptr, .ElemTy: nullptr, .IsSigned: false, .IsVolatile: false};
4044	AtomicOrdering AO = AtomicOrdering::Monotonic;
4045	ConstantInt *Expr = ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 1U);
4046	ConstantInt *D = ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 1U);
4047	OMPAtomicCompareOp OpMax = OMPAtomicCompareOp::MAX;
4048	OMPAtomicCompareOp OpEQ = OMPAtomicCompareOp::EQ;
4049
4050	Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
4051	Loc: Builder, X&: XSigned, V, R, E: Expr, D: nullptr, AO, Op: OpMax, IsXBinopExpr: true, IsPostfixUpdate: false, IsFailOnly: false));
4052	Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
4053	Loc: Builder, X&: XUnsigned, V, R, E: Expr, D: nullptr, AO, Op: OpMax, IsXBinopExpr: false, IsPostfixUpdate: false, IsFailOnly: false));
4054	Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
4055	Loc: Builder, X&: XSigned, V, R, E: Expr, D, AO, Op: OpEQ, IsXBinopExpr: true, IsPostfixUpdate: false, IsFailOnly: false));
4056
4057	BasicBlock *EntryBB = BB;
4058	EXPECT_EQ(EntryBB->getParent()->size(), 1U);
4059	EXPECT_EQ(EntryBB->size(), 5U);
4060
4061	AtomicRMWInst *ARWM1 = dyn_cast<AtomicRMWInst>(Val: Init->getNextNode());
4062	EXPECT_NE(ARWM1, nullptr);
4063	EXPECT_EQ(ARWM1->getPointerOperand(), XVal);
4064	EXPECT_EQ(ARWM1->getValOperand(), Expr);
4065	EXPECT_EQ(ARWM1->getOperation(), AtomicRMWInst::Min);
4066
4067	AtomicRMWInst *ARWM2 = dyn_cast<AtomicRMWInst>(Val: ARWM1->getNextNode());
4068	EXPECT_NE(ARWM2, nullptr);
4069	EXPECT_EQ(ARWM2->getPointerOperand(), XVal);
4070	EXPECT_EQ(ARWM2->getValOperand(), Expr);
4071	EXPECT_EQ(ARWM2->getOperation(), AtomicRMWInst::UMax);
4072
4073	AtomicCmpXchgInst *AXCHG = dyn_cast<AtomicCmpXchgInst>(Val: ARWM2->getNextNode());
4074	EXPECT_NE(AXCHG, nullptr);
4075	EXPECT_EQ(AXCHG->getPointerOperand(), XVal);
4076	EXPECT_EQ(AXCHG->getCompareOperand(), Expr);
4077	EXPECT_EQ(AXCHG->getNewValOperand(), D);
4078
4079	Builder.CreateRetVoid();
4080	OMPBuilder.finalize();
4081	EXPECT_FALSE(verifyModule(*M, &errs()));
4082	}
4083
4084	TEST_F(OpenMPIRBuilderTest, OMPAtomicCompareCapture) {
4085	OpenMPIRBuilder OMPBuilder(*M);
4086	OMPBuilder.initialize();
4087	F->setName("func");
4088	IRBuilder<> Builder(BB);
4089
4090	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4091
4092	LLVMContext &Ctx = M->getContext();
4093	IntegerType *Int32 = Type::getInt32Ty(C&: Ctx);
4094	AllocaInst *XVal = Builder.CreateAlloca(Ty: Int32);
4095	XVal->setName("x");
4096	AllocaInst *VVal = Builder.CreateAlloca(Ty: Int32);
4097	VVal->setName("v");
4098	AllocaInst *RVal = Builder.CreateAlloca(Ty: Int32);
4099	RVal->setName("r");
4100
4101	StoreInst *Init =
4102	Builder.CreateStore(Val: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 0U), Ptr: XVal);
4103
4104	OpenMPIRBuilder::AtomicOpValue X = {.Var: XVal, .ElemTy: Int32, .IsSigned: true, .IsVolatile: false};
4105	OpenMPIRBuilder::AtomicOpValue V = {.Var: VVal, .ElemTy: Int32, .IsSigned: false, .IsVolatile: false};
4106	OpenMPIRBuilder::AtomicOpValue NoV = {.Var: nullptr, .ElemTy: nullptr, .IsSigned: false, .IsVolatile: false};
4107	OpenMPIRBuilder::AtomicOpValue R = {.Var: RVal, .ElemTy: Int32, .IsSigned: false, .IsVolatile: false};
4108	OpenMPIRBuilder::AtomicOpValue NoR = {.Var: nullptr, .ElemTy: nullptr, .IsSigned: false, .IsVolatile: false};
4109
4110	AtomicOrdering AO = AtomicOrdering::Monotonic;
4111	ConstantInt *Expr = ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 1U);
4112	ConstantInt *D = ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 1U);
4113	OMPAtomicCompareOp OpMax = OMPAtomicCompareOp::MAX;
4114	OMPAtomicCompareOp OpEQ = OMPAtomicCompareOp::EQ;
4115
4116	// { cond-update-stmt v = x; }
4117	Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
4118	Loc: Builder, X, V, R&: NoR, E: Expr, D, AO, Op: OpEQ, /* IsXBinopExpr */ true,
4119	/* IsPostfixUpdate */ false,
4120	/* IsFailOnly */ false));
4121	// { v = x; cond-update-stmt }
4122	Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
4123	Loc: Builder, X, V, R&: NoR, E: Expr, D, AO, Op: OpEQ, /* IsXBinopExpr */ true,
4124	/* IsPostfixUpdate */ true,
4125	/* IsFailOnly */ false));
4126	// if(x == e) { x = d; } else { v = x; }
4127	Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
4128	Loc: Builder, X, V, R&: NoR, E: Expr, D, AO, Op: OpEQ, /* IsXBinopExpr */ true,
4129	/* IsPostfixUpdate */ false,
4130	/* IsFailOnly */ true));
4131	// { r = x == e; if(r) { x = d; } }
4132	Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
4133	Loc: Builder, X, V&: NoV, R, E: Expr, D, AO, Op: OpEQ, /* IsXBinopExpr */ true,
4134	/* IsPostfixUpdate */ false,
4135	/* IsFailOnly */ false));
4136	// { r = x == e; if(r) { x = d; } else { v = x; } }
4137	Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
4138	Loc: Builder, X, V, R, E: Expr, D, AO, Op: OpEQ, /* IsXBinopExpr */ true,
4139	/* IsPostfixUpdate */ false,
4140	/* IsFailOnly */ true));
4141
4142	// { v = x; cond-update-stmt }
4143	Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
4144	Loc: Builder, X, V, R&: NoR, E: Expr, D: nullptr, AO, Op: OpMax, /* IsXBinopExpr */ true,
4145	/* IsPostfixUpdate */ true,
4146	/* IsFailOnly */ false));
4147	// { cond-update-stmt v = x; }
4148	Builder.restoreIP(IP: OMPBuilder.createAtomicCompare(
4149	Loc: Builder, X, V, R&: NoR, E: Expr, D: nullptr, AO, Op: OpMax, /* IsXBinopExpr */ false,
4150	/* IsPostfixUpdate */ false,
4151	/* IsFailOnly */ false));
4152
4153	BasicBlock *EntryBB = BB;
4154	EXPECT_EQ(EntryBB->getParent()->size(), 5U);
4155	BasicBlock *Cont1 = dyn_cast<BasicBlock>(Val: EntryBB->getNextNode());
4156	EXPECT_NE(Cont1, nullptr);
4157	BasicBlock *Exit1 = dyn_cast<BasicBlock>(Val: Cont1->getNextNode());
4158	EXPECT_NE(Exit1, nullptr);
4159	BasicBlock *Cont2 = dyn_cast<BasicBlock>(Val: Exit1->getNextNode());
4160	EXPECT_NE(Cont2, nullptr);
4161	BasicBlock *Exit2 = dyn_cast<BasicBlock>(Val: Cont2->getNextNode());
4162	EXPECT_NE(Exit2, nullptr);
4163
4164	AtomicCmpXchgInst *CmpXchg1 =
4165	dyn_cast<AtomicCmpXchgInst>(Val: Init->getNextNode());
4166	EXPECT_NE(CmpXchg1, nullptr);
4167	EXPECT_EQ(CmpXchg1->getPointerOperand(), XVal);
4168	EXPECT_EQ(CmpXchg1->getCompareOperand(), Expr);
4169	EXPECT_EQ(CmpXchg1->getNewValOperand(), D);
4170	ExtractValueInst *ExtVal1 =
4171	dyn_cast<ExtractValueInst>(Val: CmpXchg1->getNextNode());
4172	EXPECT_NE(ExtVal1, nullptr);
4173	EXPECT_EQ(ExtVal1->getAggregateOperand(), CmpXchg1);
4174	EXPECT_EQ(ExtVal1->getIndices(), ArrayRef<unsigned int>(0U));
4175	ExtractValueInst *ExtVal2 =
4176	dyn_cast<ExtractValueInst>(Val: ExtVal1->getNextNode());
4177	EXPECT_NE(ExtVal2, nullptr);
4178	EXPECT_EQ(ExtVal2->getAggregateOperand(), CmpXchg1);
4179	EXPECT_EQ(ExtVal2->getIndices(), ArrayRef<unsigned int>(1U));
4180	SelectInst *Sel1 = dyn_cast<SelectInst>(Val: ExtVal2->getNextNode());
4181	EXPECT_NE(Sel1, nullptr);
4182	EXPECT_EQ(Sel1->getCondition(), ExtVal2);
4183	EXPECT_EQ(Sel1->getTrueValue(), Expr);
4184	EXPECT_EQ(Sel1->getFalseValue(), ExtVal1);
4185	StoreInst *Store1 = dyn_cast<StoreInst>(Val: Sel1->getNextNode());
4186	EXPECT_NE(Store1, nullptr);
4187	EXPECT_EQ(Store1->getPointerOperand(), VVal);
4188	EXPECT_EQ(Store1->getValueOperand(), Sel1);
4189
4190	AtomicCmpXchgInst *CmpXchg2 =
4191	dyn_cast<AtomicCmpXchgInst>(Val: Store1->getNextNode());
4192	EXPECT_NE(CmpXchg2, nullptr);
4193	EXPECT_EQ(CmpXchg2->getPointerOperand(), XVal);
4194	EXPECT_EQ(CmpXchg2->getCompareOperand(), Expr);
4195	EXPECT_EQ(CmpXchg2->getNewValOperand(), D);
4196	ExtractValueInst *ExtVal3 =
4197	dyn_cast<ExtractValueInst>(Val: CmpXchg2->getNextNode());
4198	EXPECT_NE(ExtVal3, nullptr);
4199	EXPECT_EQ(ExtVal3->getAggregateOperand(), CmpXchg2);
4200	EXPECT_EQ(ExtVal3->getIndices(), ArrayRef<unsigned int>(0U));
4201	StoreInst *Store2 = dyn_cast<StoreInst>(Val: ExtVal3->getNextNode());
4202	EXPECT_NE(Store2, nullptr);
4203	EXPECT_EQ(Store2->getPointerOperand(), VVal);
4204	EXPECT_EQ(Store2->getValueOperand(), ExtVal3);
4205
4206	AtomicCmpXchgInst *CmpXchg3 =
4207	dyn_cast<AtomicCmpXchgInst>(Val: Store2->getNextNode());
4208	EXPECT_NE(CmpXchg3, nullptr);
4209	EXPECT_EQ(CmpXchg3->getPointerOperand(), XVal);
4210	EXPECT_EQ(CmpXchg3->getCompareOperand(), Expr);
4211	EXPECT_EQ(CmpXchg3->getNewValOperand(), D);
4212	ExtractValueInst *ExtVal4 =
4213	dyn_cast<ExtractValueInst>(Val: CmpXchg3->getNextNode());
4214	EXPECT_NE(ExtVal4, nullptr);
4215	EXPECT_EQ(ExtVal4->getAggregateOperand(), CmpXchg3);
4216	EXPECT_EQ(ExtVal4->getIndices(), ArrayRef<unsigned int>(0U));
4217	ExtractValueInst *ExtVal5 =
4218	dyn_cast<ExtractValueInst>(Val: ExtVal4->getNextNode());
4219	EXPECT_NE(ExtVal5, nullptr);
4220	EXPECT_EQ(ExtVal5->getAggregateOperand(), CmpXchg3);
4221	EXPECT_EQ(ExtVal5->getIndices(), ArrayRef<unsigned int>(1U));
4222	BranchInst *Br1 = dyn_cast<BranchInst>(Val: ExtVal5->getNextNode());
4223	EXPECT_NE(Br1, nullptr);
4224	EXPECT_EQ(Br1->isConditional(), true);
4225	EXPECT_EQ(Br1->getCondition(), ExtVal5);
4226	EXPECT_EQ(Br1->getSuccessor(0), Exit1);
4227	EXPECT_EQ(Br1->getSuccessor(1), Cont1);
4228
4229	StoreInst *Store3 = dyn_cast<StoreInst>(Val: &Cont1->front());
4230	EXPECT_NE(Store3, nullptr);
4231	EXPECT_EQ(Store3->getPointerOperand(), VVal);
4232	EXPECT_EQ(Store3->getValueOperand(), ExtVal4);
4233	BranchInst *Br2 = dyn_cast<BranchInst>(Val: Store3->getNextNode());
4234	EXPECT_NE(Br2, nullptr);
4235	EXPECT_EQ(Br2->isUnconditional(), true);
4236	EXPECT_EQ(Br2->getSuccessor(0), Exit1);
4237
4238	AtomicCmpXchgInst *CmpXchg4 = dyn_cast<AtomicCmpXchgInst>(Val: &Exit1->front());
4239	EXPECT_NE(CmpXchg4, nullptr);
4240	EXPECT_EQ(CmpXchg4->getPointerOperand(), XVal);
4241	EXPECT_EQ(CmpXchg4->getCompareOperand(), Expr);
4242	EXPECT_EQ(CmpXchg4->getNewValOperand(), D);
4243	ExtractValueInst *ExtVal6 =
4244	dyn_cast<ExtractValueInst>(Val: CmpXchg4->getNextNode());
4245	EXPECT_NE(ExtVal6, nullptr);
4246	EXPECT_EQ(ExtVal6->getAggregateOperand(), CmpXchg4);
4247	EXPECT_EQ(ExtVal6->getIndices(), ArrayRef<unsigned int>(1U));
4248	ZExtInst *ZExt1 = dyn_cast<ZExtInst>(Val: ExtVal6->getNextNode());
4249	EXPECT_NE(ZExt1, nullptr);
4250	EXPECT_EQ(ZExt1->getDestTy(), Int32);
4251	StoreInst *Store4 = dyn_cast<StoreInst>(Val: ZExt1->getNextNode());
4252	EXPECT_NE(Store4, nullptr);
4253	EXPECT_EQ(Store4->getPointerOperand(), RVal);
4254	EXPECT_EQ(Store4->getValueOperand(), ZExt1);
4255
4256	AtomicCmpXchgInst *CmpXchg5 =
4257	dyn_cast<AtomicCmpXchgInst>(Val: Store4->getNextNode());
4258	EXPECT_NE(CmpXchg5, nullptr);
4259	EXPECT_EQ(CmpXchg5->getPointerOperand(), XVal);
4260	EXPECT_EQ(CmpXchg5->getCompareOperand(), Expr);
4261	EXPECT_EQ(CmpXchg5->getNewValOperand(), D);
4262	ExtractValueInst *ExtVal7 =
4263	dyn_cast<ExtractValueInst>(Val: CmpXchg5->getNextNode());
4264	EXPECT_NE(ExtVal7, nullptr);
4265	EXPECT_EQ(ExtVal7->getAggregateOperand(), CmpXchg5);
4266	EXPECT_EQ(ExtVal7->getIndices(), ArrayRef<unsigned int>(0U));
4267	ExtractValueInst *ExtVal8 =
4268	dyn_cast<ExtractValueInst>(Val: ExtVal7->getNextNode());
4269	EXPECT_NE(ExtVal8, nullptr);
4270	EXPECT_EQ(ExtVal8->getAggregateOperand(), CmpXchg5);
4271	EXPECT_EQ(ExtVal8->getIndices(), ArrayRef<unsigned int>(1U));
4272	BranchInst *Br3 = dyn_cast<BranchInst>(Val: ExtVal8->getNextNode());
4273	EXPECT_NE(Br3, nullptr);
4274	EXPECT_EQ(Br3->isConditional(), true);
4275	EXPECT_EQ(Br3->getCondition(), ExtVal8);
4276	EXPECT_EQ(Br3->getSuccessor(0), Exit2);
4277	EXPECT_EQ(Br3->getSuccessor(1), Cont2);
4278
4279	StoreInst *Store5 = dyn_cast<StoreInst>(Val: &Cont2->front());
4280	EXPECT_NE(Store5, nullptr);
4281	EXPECT_EQ(Store5->getPointerOperand(), VVal);
4282	EXPECT_EQ(Store5->getValueOperand(), ExtVal7);
4283	BranchInst *Br4 = dyn_cast<BranchInst>(Val: Store5->getNextNode());
4284	EXPECT_NE(Br4, nullptr);
4285	EXPECT_EQ(Br4->isUnconditional(), true);
4286	EXPECT_EQ(Br4->getSuccessor(0), Exit2);
4287
4288	ExtractValueInst *ExtVal9 = dyn_cast<ExtractValueInst>(Val: &Exit2->front());
4289	EXPECT_NE(ExtVal9, nullptr);
4290	EXPECT_EQ(ExtVal9->getAggregateOperand(), CmpXchg5);
4291	EXPECT_EQ(ExtVal9->getIndices(), ArrayRef<unsigned int>(1U));
4292	ZExtInst *ZExt2 = dyn_cast<ZExtInst>(Val: ExtVal9->getNextNode());
4293	EXPECT_NE(ZExt2, nullptr);
4294	EXPECT_EQ(ZExt2->getDestTy(), Int32);
4295	StoreInst *Store6 = dyn_cast<StoreInst>(Val: ZExt2->getNextNode());
4296	EXPECT_NE(Store6, nullptr);
4297	EXPECT_EQ(Store6->getPointerOperand(), RVal);
4298	EXPECT_EQ(Store6->getValueOperand(), ZExt2);
4299
4300	AtomicRMWInst *ARWM1 = dyn_cast<AtomicRMWInst>(Val: Store6->getNextNode());
4301	EXPECT_NE(ARWM1, nullptr);
4302	EXPECT_EQ(ARWM1->getPointerOperand(), XVal);
4303	EXPECT_EQ(ARWM1->getValOperand(), Expr);
4304	EXPECT_EQ(ARWM1->getOperation(), AtomicRMWInst::Min);
4305	StoreInst *Store7 = dyn_cast<StoreInst>(Val: ARWM1->getNextNode());
4306	EXPECT_NE(Store7, nullptr);
4307	EXPECT_EQ(Store7->getPointerOperand(), VVal);
4308	EXPECT_EQ(Store7->getValueOperand(), ARWM1);
4309
4310	AtomicRMWInst *ARWM2 = dyn_cast<AtomicRMWInst>(Val: Store7->getNextNode());
4311	EXPECT_NE(ARWM2, nullptr);
4312	EXPECT_EQ(ARWM2->getPointerOperand(), XVal);
4313	EXPECT_EQ(ARWM2->getValOperand(), Expr);
4314	EXPECT_EQ(ARWM2->getOperation(), AtomicRMWInst::Max);
4315	CmpInst *Cmp1 = dyn_cast<CmpInst>(Val: ARWM2->getNextNode());
4316	EXPECT_NE(Cmp1, nullptr);
4317	EXPECT_EQ(Cmp1->getPredicate(), CmpInst::ICMP_SGT);
4318	EXPECT_EQ(Cmp1->getOperand(0), ARWM2);
4319	EXPECT_EQ(Cmp1->getOperand(1), Expr);
4320	SelectInst *Sel2 = dyn_cast<SelectInst>(Val: Cmp1->getNextNode());
4321	EXPECT_NE(Sel2, nullptr);
4322	EXPECT_EQ(Sel2->getCondition(), Cmp1);
4323	EXPECT_EQ(Sel2->getTrueValue(), Expr);
4324	EXPECT_EQ(Sel2->getFalseValue(), ARWM2);
4325	StoreInst *Store8 = dyn_cast<StoreInst>(Val: Sel2->getNextNode());
4326	EXPECT_NE(Store8, nullptr);
4327	EXPECT_EQ(Store8->getPointerOperand(), VVal);
4328	EXPECT_EQ(Store8->getValueOperand(), Sel2);
4329
4330	Builder.CreateRetVoid();
4331	OMPBuilder.finalize();
4332	EXPECT_FALSE(verifyModule(*M, &errs()));
4333	}
4334
4335	TEST_F(OpenMPIRBuilderTest, CreateTeams) {
4336	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4337	OpenMPIRBuilder OMPBuilder(*M);
4338	OMPBuilder.Config.IsTargetDevice = false;
4339	OMPBuilder.initialize();
4340	F->setName("func");
4341	IRBuilder<> Builder(BB);
4342
4343	AllocaInst *ValPtr32 = Builder.CreateAlloca(Ty: Builder.getInt32Ty());
4344	AllocaInst *ValPtr128 = Builder.CreateAlloca(Ty: Builder.getInt128Ty());
4345	Value *Val128 = Builder.CreateLoad(Ty: Builder.getInt128Ty(), Ptr: ValPtr128, Name: "load");
4346
4347	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4348	Builder.restoreIP(IP: AllocaIP);
4349	AllocaInst *Local128 = Builder.CreateAlloca(Ty: Builder.getInt128Ty(), ArraySize: nullptr,
4350	Name: "bodygen.alloca128");
4351
4352	Builder.restoreIP(IP: CodeGenIP);
4353	// Loading and storing captured pointer and values
4354	Builder.CreateStore(Val: Val128, Ptr: Local128);
4355	Value *Val32 = Builder.CreateLoad(Ty: ValPtr32->getAllocatedType(), Ptr: ValPtr32,
4356	Name: "bodygen.load32");
4357
4358	LoadInst *PrivLoad128 = Builder.CreateLoad(
4359	Ty: Local128->getAllocatedType(), Ptr: Local128, Name: "bodygen.local.load128");
4360	Value *Cmp = Builder.CreateICmpNE(
4361	LHS: Val32, RHS: Builder.CreateTrunc(V: PrivLoad128, DestTy: Val32->getType()));
4362	Instruction *ThenTerm, *ElseTerm;
4363	SplitBlockAndInsertIfThenElse(Cond: Cmp, SplitBefore: CodeGenIP.getBlock()->getTerminator(),
4364	ThenTerm: &ThenTerm, ElseTerm: &ElseTerm);
4365	};
4366
4367	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4368	Builder.restoreIP(IP: OMPBuilder.createTeams(
4369	Loc: Builder, BodyGenCB, /*NumTeamsLower=*/nullptr, /*NumTeamsUpper=*/nullptr,
4370	/*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr));
4371
4372	OMPBuilder.finalize();
4373	Builder.CreateRetVoid();
4374
4375	EXPECT_FALSE(verifyModule(*M, &errs()));
4376
4377	CallInst *TeamsForkCall = dyn_cast<CallInst>(
4378	Val: OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_fork_teams)
4379	->user_back());
4380
4381	// Verify the Ident argument
4382	GlobalVariable *Ident = cast<GlobalVariable>(Val: TeamsForkCall->getArgOperand(i: 0));
4383	ASSERT_NE(Ident, nullptr);
4384	EXPECT_TRUE(Ident->hasInitializer());
4385	Constant *Initializer = Ident->getInitializer();
4386	GlobalVariable *SrcStrGlob =
4387	cast<GlobalVariable>(Val: Initializer->getOperand(i: 4)->stripPointerCasts());
4388	ASSERT_NE(SrcStrGlob, nullptr);
4389	ConstantDataArray *SrcSrc =
4390	dyn_cast<ConstantDataArray>(Val: SrcStrGlob->getInitializer());
4391	ASSERT_NE(SrcSrc, nullptr);
4392
4393	// Verify the outlined function signature.
4394	Function *OutlinedFn =
4395	dyn_cast<Function>(Val: TeamsForkCall->getArgOperand(i: 2)->stripPointerCasts());
4396	ASSERT_NE(OutlinedFn, nullptr);
4397	EXPECT_FALSE(OutlinedFn->isDeclaration());
4398	EXPECT_TRUE(OutlinedFn->arg_size() >= 3);
4399	EXPECT_EQ(OutlinedFn->getArg(0)->getType(), Builder.getPtrTy()); // global_tid
4400	EXPECT_EQ(OutlinedFn->getArg(1)->getType(), Builder.getPtrTy()); // bound_tid
4401	EXPECT_EQ(OutlinedFn->getArg(2)->getType(),
4402	Builder.getPtrTy()); // captured args
4403
4404	// Check for TruncInst and ICmpInst in the outlined function.
4405	EXPECT_TRUE(any_of(instructions(OutlinedFn),
4406	[](Instruction &inst) { return isa<TruncInst>(&inst); }));
4407	EXPECT_TRUE(any_of(instructions(OutlinedFn),
4408	[](Instruction &inst) { return isa<ICmpInst>(&inst); }));
4409	}
4410
4411	TEST_F(OpenMPIRBuilderTest, CreateTeamsWithThreadLimit) {
4412	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4413	OpenMPIRBuilder OMPBuilder(*M);
4414	OMPBuilder.Config.IsTargetDevice = false;
4415	OMPBuilder.initialize();
4416	F->setName("func");
4417	IRBuilder<> &Builder = OMPBuilder.Builder;
4418	Builder.SetInsertPoint(BB);
4419
4420	Function *FakeFunction =
4421	Function::Create(Ty: FunctionType::get(Result: Builder.getVoidTy(), isVarArg: false),
4422	Linkage: GlobalValue::ExternalLinkage, N: "fakeFunction", M: M.get());
4423
4424	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4425	Builder.restoreIP(IP: CodeGenIP);
4426	Builder.CreateCall(Callee: FakeFunction, Args: {});
4427	};
4428
4429	// `F` has an argument - an integer, so we use that as the thread limit.
4430	Builder.restoreIP(IP: OMPBuilder.createTeams(/*=*/Loc: Builder, BodyGenCB,
4431	/*NumTeamsLower=*/nullptr,
4432	/*NumTeamsUpper=*/nullptr,
4433	/*ThreadLimit=*/F->arg_begin(),
4434	/*IfExpr=*/nullptr));
4435
4436	Builder.CreateRetVoid();
4437	OMPBuilder.finalize();
4438
4439	ASSERT_FALSE(verifyModule(*M));
4440
4441	CallInst *PushNumTeamsCallInst =
4442	findSingleCall(F, FnID: OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
4443	ASSERT_NE(PushNumTeamsCallInst, nullptr);
4444
4445	EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), Builder.getInt32(0));
4446	EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), Builder.getInt32(0));
4447	EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), &*F->arg_begin());
4448
4449	// Verifying that the next instruction to execute is kmpc_fork_teams
4450	BranchInst *BrInst =
4451	dyn_cast<BranchInst>(Val: PushNumTeamsCallInst->getNextNonDebugInstruction());
4452	ASSERT_NE(BrInst, nullptr);
4453	ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
4454	Instruction *NextInstruction =
4455	BrInst->getSuccessor(i: 0)->getFirstNonPHIOrDbgOrLifetime();
4456	CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(Val: NextInstruction);
4457	ASSERT_NE(ForkTeamsCI, nullptr);
4458	EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
4459	OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
4460	}
4461
4462	TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsUpper) {
4463	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4464	OpenMPIRBuilder OMPBuilder(*M);
4465	OMPBuilder.Config.IsTargetDevice = false;
4466	OMPBuilder.initialize();
4467	F->setName("func");
4468	IRBuilder<> &Builder = OMPBuilder.Builder;
4469	Builder.SetInsertPoint(BB);
4470
4471	Function *FakeFunction =
4472	Function::Create(Ty: FunctionType::get(Result: Builder.getVoidTy(), isVarArg: false),
4473	Linkage: GlobalValue::ExternalLinkage, N: "fakeFunction", M: M.get());
4474
4475	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4476	Builder.restoreIP(IP: CodeGenIP);
4477	Builder.CreateCall(Callee: FakeFunction, Args: {});
4478	};
4479
4480	// `F` already has an integer argument, so we use that as upper bound to
4481	// `num_teams`
4482	Builder.restoreIP(IP: OMPBuilder.createTeams(Loc: Builder, BodyGenCB,
4483	/*NumTeamsLower=*/nullptr,
4484	/*NumTeamsUpper=*/F->arg_begin(),
4485	/*ThreadLimit=*/nullptr,
4486	/*IfExpr=*/nullptr));
4487
4488	Builder.CreateRetVoid();
4489	OMPBuilder.finalize();
4490
4491	ASSERT_FALSE(verifyModule(*M));
4492
4493	CallInst *PushNumTeamsCallInst =
4494	findSingleCall(F, FnID: OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
4495	ASSERT_NE(PushNumTeamsCallInst, nullptr);
4496
4497	EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), &*F->arg_begin());
4498	EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), &*F->arg_begin());
4499	EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), Builder.getInt32(0));
4500
4501	// Verifying that the next instruction to execute is kmpc_fork_teams
4502	BranchInst *BrInst =
4503	dyn_cast<BranchInst>(Val: PushNumTeamsCallInst->getNextNonDebugInstruction());
4504	ASSERT_NE(BrInst, nullptr);
4505	ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
4506	Instruction *NextInstruction =
4507	BrInst->getSuccessor(i: 0)->getFirstNonPHIOrDbgOrLifetime();
4508	CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(Val: NextInstruction);
4509	ASSERT_NE(ForkTeamsCI, nullptr);
4510	EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
4511	OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
4512	}
4513
4514	TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsBoth) {
4515	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4516	OpenMPIRBuilder OMPBuilder(*M);
4517	OMPBuilder.Config.IsTargetDevice = false;
4518	OMPBuilder.initialize();
4519	F->setName("func");
4520	IRBuilder<> &Builder = OMPBuilder.Builder;
4521	Builder.SetInsertPoint(BB);
4522
4523	Function *FakeFunction =
4524	Function::Create(Ty: FunctionType::get(Result: Builder.getVoidTy(), isVarArg: false),
4525	Linkage: GlobalValue::ExternalLinkage, N: "fakeFunction", M: M.get());
4526
4527	Value *NumTeamsLower =
4528	Builder.CreateAdd(LHS: F->arg_begin(), RHS: Builder.getInt32(C: 5), Name: "numTeamsLower");
4529	Value *NumTeamsUpper =
4530	Builder.CreateAdd(LHS: F->arg_begin(), RHS: Builder.getInt32(C: 10), Name: "numTeamsUpper");
4531
4532	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4533	Builder.restoreIP(IP: CodeGenIP);
4534	Builder.CreateCall(Callee: FakeFunction, Args: {});
4535	};
4536
4537	// `F` already has an integer argument, so we use that as upper bound to
4538	// `num_teams`
4539	Builder.restoreIP(
4540	IP: OMPBuilder.createTeams(Loc: Builder, BodyGenCB, NumTeamsLower, NumTeamsUpper,
4541	/*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr));
4542
4543	Builder.CreateRetVoid();
4544	OMPBuilder.finalize();
4545
4546	ASSERT_FALSE(verifyModule(*M));
4547
4548	CallInst *PushNumTeamsCallInst =
4549	findSingleCall(F, FnID: OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
4550	ASSERT_NE(PushNumTeamsCallInst, nullptr);
4551
4552	EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), NumTeamsLower);
4553	EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), NumTeamsUpper);
4554	EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), Builder.getInt32(0));
4555
4556	// Verifying that the next instruction to execute is kmpc_fork_teams
4557	BranchInst *BrInst =
4558	dyn_cast<BranchInst>(Val: PushNumTeamsCallInst->getNextNonDebugInstruction());
4559	ASSERT_NE(BrInst, nullptr);
4560	ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
4561	Instruction *NextInstruction =
4562	BrInst->getSuccessor(i: 0)->getFirstNonPHIOrDbgOrLifetime();
4563	CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(Val: NextInstruction);
4564	ASSERT_NE(ForkTeamsCI, nullptr);
4565	EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
4566	OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
4567	}
4568
4569	TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsAndThreadLimit) {
4570	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4571	OpenMPIRBuilder OMPBuilder(*M);
4572	OMPBuilder.Config.IsTargetDevice = false;
4573	OMPBuilder.initialize();
4574	F->setName("func");
4575	IRBuilder<> &Builder = OMPBuilder.Builder;
4576	Builder.SetInsertPoint(BB);
4577
4578	BasicBlock *CodegenBB = splitBB(Builder, CreateBranch: true);
4579	Builder.SetInsertPoint(CodegenBB);
4580
4581	// Generate values for `num_teams` and `thread_limit` using the first argument
4582	// of the testing function.
4583	Value *NumTeamsLower =
4584	Builder.CreateAdd(LHS: F->arg_begin(), RHS: Builder.getInt32(C: 5), Name: "numTeamsLower");
4585	Value *NumTeamsUpper =
4586	Builder.CreateAdd(LHS: F->arg_begin(), RHS: Builder.getInt32(C: 10), Name: "numTeamsUpper");
4587	Value *ThreadLimit =
4588	Builder.CreateAdd(LHS: F->arg_begin(), RHS: Builder.getInt32(C: 20), Name: "threadLimit");
4589
4590	Function *FakeFunction =
4591	Function::Create(Ty: FunctionType::get(Result: Builder.getVoidTy(), isVarArg: false),
4592	Linkage: GlobalValue::ExternalLinkage, N: "fakeFunction", M: M.get());
4593
4594	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4595	Builder.restoreIP(IP: CodeGenIP);
4596	Builder.CreateCall(Callee: FakeFunction, Args: {});
4597	};
4598
4599	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4600	Builder.restoreIP(IP: OMPBuilder.createTeams(
4601	Loc: Builder, BodyGenCB, NumTeamsLower, NumTeamsUpper, ThreadLimit, IfExpr: nullptr));
4602
4603	Builder.CreateRetVoid();
4604	OMPBuilder.finalize();
4605
4606	ASSERT_FALSE(verifyModule(*M));
4607
4608	CallInst *PushNumTeamsCallInst =
4609	findSingleCall(F, FnID: OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
4610	ASSERT_NE(PushNumTeamsCallInst, nullptr);
4611
4612	EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), NumTeamsLower);
4613	EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), NumTeamsUpper);
4614	EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), ThreadLimit);
4615
4616	// Verifying that the next instruction to execute is kmpc_fork_teams
4617	BranchInst *BrInst =
4618	dyn_cast<BranchInst>(Val: PushNumTeamsCallInst->getNextNonDebugInstruction());
4619	ASSERT_NE(BrInst, nullptr);
4620	ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
4621	Instruction *NextInstruction =
4622	BrInst->getSuccessor(i: 0)->getFirstNonPHIOrDbgOrLifetime();
4623	CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(Val: NextInstruction);
4624	ASSERT_NE(ForkTeamsCI, nullptr);
4625	EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
4626	OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
4627	}
4628
4629	TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfCondition) {
4630	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4631	OpenMPIRBuilder OMPBuilder(*M);
4632	OMPBuilder.Config.IsTargetDevice = false;
4633	OMPBuilder.initialize();
4634	F->setName("func");
4635	IRBuilder<> &Builder = OMPBuilder.Builder;
4636	Builder.SetInsertPoint(BB);
4637
4638	Value *IfExpr = Builder.CreateLoad(Ty: Builder.getInt1Ty(),
4639	Ptr: Builder.CreateAlloca(Ty: Builder.getInt1Ty()));
4640
4641	Function *FakeFunction =
4642	Function::Create(Ty: FunctionType::get(Result: Builder.getVoidTy(), isVarArg: false),
4643	Linkage: GlobalValue::ExternalLinkage, N: "fakeFunction", M: M.get());
4644
4645	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4646	Builder.restoreIP(IP: CodeGenIP);
4647	Builder.CreateCall(Callee: FakeFunction, Args: {});
4648	};
4649
4650	// `F` already has an integer argument, so we use that as upper bound to
4651	// `num_teams`
4652	Builder.restoreIP(IP: OMPBuilder.createTeams(
4653	Loc: Builder, BodyGenCB, /*NumTeamsLower=*/nullptr, /*NumTeamsUpper=*/nullptr,
4654	/*ThreadLimit=*/nullptr, IfExpr));
4655
4656	Builder.CreateRetVoid();
4657	OMPBuilder.finalize();
4658
4659	ASSERT_FALSE(verifyModule(*M));
4660
4661	CallInst *PushNumTeamsCallInst =
4662	findSingleCall(F, FnID: OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
4663	ASSERT_NE(PushNumTeamsCallInst, nullptr);
4664	Value *NumTeamsLower = PushNumTeamsCallInst->getArgOperand(i: 2);
4665	Value *NumTeamsUpper = PushNumTeamsCallInst->getArgOperand(i: 3);
4666	Value *ThreadLimit = PushNumTeamsCallInst->getArgOperand(i: 4);
4667
4668	// Check the lower_bound
4669	ASSERT_NE(NumTeamsLower, nullptr);
4670	SelectInst *NumTeamsLowerSelectInst = dyn_cast<SelectInst>(Val: NumTeamsLower);
4671	ASSERT_NE(NumTeamsLowerSelectInst, nullptr);
4672	EXPECT_EQ(NumTeamsLowerSelectInst->getCondition(), IfExpr);
4673	EXPECT_EQ(NumTeamsLowerSelectInst->getTrueValue(), Builder.getInt32(0));
4674	EXPECT_EQ(NumTeamsLowerSelectInst->getFalseValue(), Builder.getInt32(1));
4675
4676	// Check the upper_bound
4677	ASSERT_NE(NumTeamsUpper, nullptr);
4678	SelectInst *NumTeamsUpperSelectInst = dyn_cast<SelectInst>(Val: NumTeamsUpper);
4679	ASSERT_NE(NumTeamsUpperSelectInst, nullptr);
4680	EXPECT_EQ(NumTeamsUpperSelectInst->getCondition(), IfExpr);
4681	EXPECT_EQ(NumTeamsUpperSelectInst->getTrueValue(), Builder.getInt32(0));
4682	EXPECT_EQ(NumTeamsUpperSelectInst->getFalseValue(), Builder.getInt32(1));
4683
4684	// Check thread_limit
4685	EXPECT_EQ(ThreadLimit, Builder.getInt32(0));
4686	}
4687
4688	TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfConditionAndNumTeams) {
4689	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4690	OpenMPIRBuilder OMPBuilder(*M);
4691	OMPBuilder.Config.IsTargetDevice = false;
4692	OMPBuilder.initialize();
4693	F->setName("func");
4694	IRBuilder<> &Builder = OMPBuilder.Builder;
4695	Builder.SetInsertPoint(BB);
4696
4697	Value *IfExpr = Builder.CreateLoad(
4698	Ty: Builder.getInt32Ty(), Ptr: Builder.CreateAlloca(Ty: Builder.getInt32Ty()));
4699	Value *NumTeamsLower = Builder.CreateAdd(LHS: F->arg_begin(), RHS: Builder.getInt32(C: 5));
4700	Value *NumTeamsUpper =
4701	Builder.CreateAdd(LHS: F->arg_begin(), RHS: Builder.getInt32(C: 10));
4702	Value *ThreadLimit = Builder.CreateAdd(LHS: F->arg_begin(), RHS: Builder.getInt32(C: 20));
4703
4704	Function *FakeFunction =
4705	Function::Create(Ty: FunctionType::get(Result: Builder.getVoidTy(), isVarArg: false),
4706	Linkage: GlobalValue::ExternalLinkage, N: "fakeFunction", M: M.get());
4707
4708	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4709	Builder.restoreIP(IP: CodeGenIP);
4710	Builder.CreateCall(Callee: FakeFunction, Args: {});
4711	};
4712
4713	// `F` already has an integer argument, so we use that as upper bound to
4714	// `num_teams`
4715	Builder.restoreIP(IP: OMPBuilder.createTeams(Loc: Builder, BodyGenCB, NumTeamsLower,
4716	NumTeamsUpper, ThreadLimit, IfExpr));
4717
4718	Builder.CreateRetVoid();
4719	OMPBuilder.finalize();
4720
4721	ASSERT_FALSE(verifyModule(*M));
4722
4723	CallInst *PushNumTeamsCallInst =
4724	findSingleCall(F, FnID: OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
4725	ASSERT_NE(PushNumTeamsCallInst, nullptr);
4726	Value *NumTeamsLowerArg = PushNumTeamsCallInst->getArgOperand(i: 2);
4727	Value *NumTeamsUpperArg = PushNumTeamsCallInst->getArgOperand(i: 3);
4728	Value *ThreadLimitArg = PushNumTeamsCallInst->getArgOperand(i: 4);
4729
4730	// Get the boolean conversion of if expression
4731	ASSERT_EQ(IfExpr->getNumUses(), 1U);
4732	User *IfExprInst = IfExpr->user_back();
4733	ICmpInst *IfExprCmpInst = dyn_cast<ICmpInst>(Val: IfExprInst);
4734	ASSERT_NE(IfExprCmpInst, nullptr);
4735	EXPECT_EQ(IfExprCmpInst->getPredicate(), ICmpInst::Predicate::ICMP_NE);
4736	EXPECT_EQ(IfExprCmpInst->getOperand(0), IfExpr);
4737	EXPECT_EQ(IfExprCmpInst->getOperand(1), Builder.getInt32(0));
4738
4739	// Check the lower_bound
4740	ASSERT_NE(NumTeamsLowerArg, nullptr);
4741	SelectInst *NumTeamsLowerSelectInst = dyn_cast<SelectInst>(Val: NumTeamsLowerArg);
4742	ASSERT_NE(NumTeamsLowerSelectInst, nullptr);
4743	EXPECT_EQ(NumTeamsLowerSelectInst->getCondition(), IfExprCmpInst);
4744	EXPECT_EQ(NumTeamsLowerSelectInst->getTrueValue(), NumTeamsLower);
4745	EXPECT_EQ(NumTeamsLowerSelectInst->getFalseValue(), Builder.getInt32(1));
4746
4747	// Check the upper_bound
4748	ASSERT_NE(NumTeamsUpperArg, nullptr);
4749	SelectInst *NumTeamsUpperSelectInst = dyn_cast<SelectInst>(Val: NumTeamsUpperArg);
4750	ASSERT_NE(NumTeamsUpperSelectInst, nullptr);
4751	EXPECT_EQ(NumTeamsUpperSelectInst->getCondition(), IfExprCmpInst);
4752	EXPECT_EQ(NumTeamsUpperSelectInst->getTrueValue(), NumTeamsUpper);
4753	EXPECT_EQ(NumTeamsUpperSelectInst->getFalseValue(), Builder.getInt32(1));
4754
4755	// Check thread_limit
4756	EXPECT_EQ(ThreadLimitArg, ThreadLimit);
4757	}
4758
4759	/// Returns the single instruction of InstTy type in BB that uses the value V.
4760	/// If there is more than one such instruction, returns null.
4761	template <typename InstTy>
4762	static InstTy *findSingleUserInBlock(Value *V, BasicBlock *BB) {
4763	InstTy *Result = nullptr;
4764	for (User *U : V->users()) {
4765	auto *Inst = dyn_cast<InstTy>(U);
4766	if (!Inst || Inst->getParent() != BB)
4767	continue;
4768	if (Result) {
4769	if (auto *SI = dyn_cast<StoreInst>(Inst)) {
4770	if (V == SI->getValueOperand())
4771	continue;
4772	} else {
4773	return nullptr;
4774	}
4775	}
4776	Result = Inst;
4777	}
4778	return Result;
4779	}
4780
4781	/// Returns true if BB contains a simple binary reduction that loads a value
4782	/// from Accum, performs some binary operation with it, and stores it back to
4783	/// Accum.
4784	static bool isSimpleBinaryReduction(Value *Accum, BasicBlock *BB,
4785	Instruction::BinaryOps *OpCode = nullptr) {
4786	StoreInst *Store = findSingleUserInBlock<StoreInst>(V: Accum, BB);
4787	if (!Store)
4788	return false;
4789	auto *Stored = dyn_cast<BinaryOperator>(Val: Store->getOperand(i_nocapture: 0));
4790	if (!Stored)
4791	return false;
4792	if (OpCode && *OpCode != Stored->getOpcode())
4793	return false;
4794	auto *Load = dyn_cast<LoadInst>(Val: Stored->getOperand(i_nocapture: 0));
4795	return Load && Load->getOperand(i_nocapture: 0) == Accum;
4796	}
4797
4798	/// Returns true if BB contains a binary reduction that reduces V using a binary
4799	/// operator into an accumulator that is a function argument.
4800	static bool isValueReducedToFuncArg(Value *V, BasicBlock *BB) {
4801	auto *ReductionOp = findSingleUserInBlock<BinaryOperator>(V, BB);
4802	if (!ReductionOp)
4803	return false;
4804
4805	auto *GlobalLoad = dyn_cast<LoadInst>(Val: ReductionOp->getOperand(i_nocapture: 0));
4806	if (!GlobalLoad)
4807	return false;
4808
4809	auto *Store = findSingleUserInBlock<StoreInst>(V: ReductionOp, BB);
4810	if (!Store)
4811	return false;
4812
4813	return Store->getPointerOperand() == GlobalLoad->getPointerOperand() &&
4814	isa<Argument>(Val: findAggregateFromValue(V: GlobalLoad->getPointerOperand()));
4815	}
4816
4817	/// Finds among users of Ptr a pair of GEP instructions with indices [0, 0] and
4818	/// [0, 1], respectively, and assigns results of these instructions to Zero and
4819	/// One. Returns true on success, false on failure or if such instructions are
4820	/// not unique among the users of Ptr.
4821	static bool findGEPZeroOne(Value *Ptr, Value *&Zero, Value *&One) {
4822	Zero = nullptr;
4823	One = nullptr;
4824	for (User *U : Ptr->users()) {
4825	if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: U)) {
4826	if (GEP->getNumIndices() != 2)
4827	continue;
4828	auto *FirstIdx = dyn_cast<ConstantInt>(Val: GEP->getOperand(i_nocapture: 1));
4829	auto *SecondIdx = dyn_cast<ConstantInt>(Val: GEP->getOperand(i_nocapture: 2));
4830	EXPECT_NE(FirstIdx, nullptr);
4831	EXPECT_NE(SecondIdx, nullptr);
4832
4833	EXPECT_TRUE(FirstIdx->isZero());
4834	if (SecondIdx->isZero()) {
4835	if (Zero)
4836	return false;
4837	Zero = GEP;
4838	} else if (SecondIdx->isOne()) {
4839	if (One)
4840	return false;
4841	One = GEP;
4842	} else {
4843	return false;
4844	}
4845	}
4846	}
4847	return Zero != nullptr && One != nullptr;
4848	}
4849
4850	static OpenMPIRBuilder::InsertPointTy
4851	sumReduction(OpenMPIRBuilder::InsertPointTy IP, Value *LHS, Value *RHS,
4852	Value *&Result) {
4853	IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
4854	Result = Builder.CreateFAdd(L: LHS, R: RHS, Name: "red.add");
4855	return Builder.saveIP();
4856	}
4857
4858	static OpenMPIRBuilder::InsertPointTy
4859	sumAtomicReduction(OpenMPIRBuilder::InsertPointTy IP, Type *Ty, Value *LHS,
4860	Value *RHS) {
4861	IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
4862	Value *Partial = Builder.CreateLoad(Ty, Ptr: RHS, Name: "red.partial");
4863	Builder.CreateAtomicRMW(Op: AtomicRMWInst::FAdd, Ptr: LHS, Val: Partial, Align: std::nullopt,
4864	Ordering: AtomicOrdering::Monotonic);
4865	return Builder.saveIP();
4866	}
4867
4868	static OpenMPIRBuilder::InsertPointTy
4869	xorReduction(OpenMPIRBuilder::InsertPointTy IP, Value *LHS, Value *RHS,
4870	Value *&Result) {
4871	IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
4872	Result = Builder.CreateXor(LHS, RHS, Name: "red.xor");
4873	return Builder.saveIP();
4874	}
4875
4876	static OpenMPIRBuilder::InsertPointTy
4877	xorAtomicReduction(OpenMPIRBuilder::InsertPointTy IP, Type *Ty, Value *LHS,
4878	Value *RHS) {
4879	IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
4880	Value *Partial = Builder.CreateLoad(Ty, Ptr: RHS, Name: "red.partial");
4881	Builder.CreateAtomicRMW(Op: AtomicRMWInst::Xor, Ptr: LHS, Val: Partial, Align: std::nullopt,
4882	Ordering: AtomicOrdering::Monotonic);
4883	return Builder.saveIP();
4884	}
4885
4886	TEST_F(OpenMPIRBuilderTest, CreateReductions) {
4887	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4888	OpenMPIRBuilder OMPBuilder(*M);
4889	OMPBuilder.Config.IsTargetDevice = false;
4890	OMPBuilder.initialize();
4891	F->setName("func");
4892	IRBuilder<> Builder(BB);
4893
4894	BasicBlock *EnterBB = BasicBlock::Create(Context&: Ctx, Name: "parallel.enter", Parent: F);
4895	Builder.CreateBr(Dest: EnterBB);
4896	Builder.SetInsertPoint(EnterBB);
4897	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4898
4899	// Create variables to be reduced.
4900	InsertPointTy OuterAllocaIP(&F->getEntryBlock(),
4901	F->getEntryBlock().getFirstInsertionPt());
4902	Type *SumType = Builder.getFloatTy();
4903	Type *XorType = Builder.getInt32Ty();
4904	Value *SumReduced;
4905	Value *XorReduced;
4906	{
4907	IRBuilderBase::InsertPointGuard Guard(Builder);
4908	Builder.restoreIP(IP: OuterAllocaIP);
4909	SumReduced = Builder.CreateAlloca(Ty: SumType);
4910	XorReduced = Builder.CreateAlloca(Ty: XorType);
4911	}
4912
4913	// Store initial values of reductions into global variables.
4914	Builder.CreateStore(Val: ConstantFP::get(Ty: Builder.getFloatTy(), V: 0.0), Ptr: SumReduced);
4915	Builder.CreateStore(Val: Builder.getInt32(C: 1), Ptr: XorReduced);
4916
4917	// The loop body computes two reductions:
4918	// sum of (float) thread-id;
4919	// xor of thread-id;
4920	// and store the result in global variables.
4921	InsertPointTy BodyIP, BodyAllocaIP;
4922	auto BodyGenCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP) {
4923	IRBuilderBase::InsertPointGuard Guard(Builder);
4924	Builder.restoreIP(IP: CodeGenIP);
4925
4926	uint32_t StrSize;
4927	Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, SrcLocStrSize&: StrSize);
4928	Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize: StrSize);
4929	Value *TID = OMPBuilder.getOrCreateThreadID(Ident);
4930	Value *SumLocal =
4931	Builder.CreateUIToFP(V: TID, DestTy: Builder.getFloatTy(), Name: "sum.local");
4932	Value *SumPartial = Builder.CreateLoad(Ty: SumType, Ptr: SumReduced, Name: "sum.partial");
4933	Value *XorPartial = Builder.CreateLoad(Ty: XorType, Ptr: XorReduced, Name: "xor.partial");
4934	Value *Sum = Builder.CreateFAdd(L: SumPartial, R: SumLocal, Name: "sum");
4935	Value *Xor = Builder.CreateXor(LHS: XorPartial, RHS: TID, Name: "xor");
4936	Builder.CreateStore(Val: Sum, Ptr: SumReduced);
4937	Builder.CreateStore(Val: Xor, Ptr: XorReduced);
4938
4939	BodyIP = Builder.saveIP();
4940	BodyAllocaIP = InnerAllocaIP;
4941	};
4942
4943	// Privatization for reduction creates local copies of reduction variables and
4944	// initializes them to reduction-neutral values.
4945	Value *SumPrivatized;
4946	Value *XorPrivatized;
4947	auto PrivCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP,
4948	Value &Original, Value &Inner, Value *&ReplVal) {
4949	IRBuilderBase::InsertPointGuard Guard(Builder);
4950	Builder.restoreIP(IP: InnerAllocaIP);
4951	if (&Original == SumReduced) {
4952	SumPrivatized = Builder.CreateAlloca(Ty: Builder.getFloatTy());
4953	ReplVal = SumPrivatized;
4954	} else if (&Original == XorReduced) {
4955	XorPrivatized = Builder.CreateAlloca(Ty: Builder.getInt32Ty());
4956	ReplVal = XorPrivatized;
4957	} else {
4958	ReplVal = &Inner;
4959	return CodeGenIP;
4960	}
4961
4962	Builder.restoreIP(IP: CodeGenIP);
4963	if (&Original == SumReduced)
4964	Builder.CreateStore(Val: ConstantFP::get(Ty: Builder.getFloatTy(), V: 0.0),
4965	Ptr: SumPrivatized);
4966	else if (&Original == XorReduced)
4967	Builder.CreateStore(Val: Builder.getInt32(C: 0), Ptr: XorPrivatized);
4968
4969	return Builder.saveIP();
4970	};
4971
4972	// Do nothing in finalization.
4973	auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; };
4974
4975	InsertPointTy AfterIP =
4976	OMPBuilder.createParallel(Loc, OuterAllocaIP, BodyGenCB, PrivCB, FiniCB,
4977	/* IfCondition */ nullptr,
4978	/* NumThreads */ nullptr, OMP_PROC_BIND_default,
4979	/* IsCancellable */ false);
4980	Builder.restoreIP(IP: AfterIP);
4981
4982	OpenMPIRBuilder::ReductionInfo ReductionInfos[] = {
4983	{SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction},
4984	{XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}};
4985
4986	OMPBuilder.createReductions(Loc: BodyIP, AllocaIP: BodyAllocaIP, ReductionInfos);
4987
4988	Builder.restoreIP(IP: AfterIP);
4989	Builder.CreateRetVoid();
4990
4991	OMPBuilder.finalize(Fn: F);
4992
4993	// The IR must be valid.
4994	EXPECT_FALSE(verifyModule(*M));
4995
4996	// Outlining must have happened.
4997	SmallVector<CallInst *> ForkCalls;
4998	findCalls(F, FnID: omp::RuntimeFunction::OMPRTL___kmpc_fork_call, OMPBuilder,
4999	Calls&: ForkCalls);
5000	ASSERT_EQ(ForkCalls.size(), 1u);
5001	Value *CalleeVal = ForkCalls[0]->getOperand(i_nocapture: 2);
5002	Function *Outlined = dyn_cast<Function>(Val: CalleeVal);
5003	EXPECT_NE(Outlined, nullptr);
5004
5005	// Check that the lock variable was created with the expected name.
5006	GlobalVariable *LockVar =
5007	M->getGlobalVariable(Name: ".gomp_critical_user_.reduction.var");
5008	EXPECT_NE(LockVar, nullptr);
5009
5010	// Find the allocation of a local array that will be used to call the runtime
5011	// reduciton function.
5012	BasicBlock &AllocBlock = Outlined->getEntryBlock();
5013	Value *LocalArray = nullptr;
5014	for (Instruction &I : AllocBlock) {
5015	if (AllocaInst *Alloc = dyn_cast<AllocaInst>(Val: &I)) {
5016	if (!Alloc->getAllocatedType()->isArrayTy() ||
5017	!Alloc->getAllocatedType()->getArrayElementType()->isPointerTy())
5018	continue;
5019	LocalArray = Alloc;
5020	break;
5021	}
5022	}
5023	ASSERT_NE(LocalArray, nullptr);
5024
5025	// Find the call to the runtime reduction function.
5026	BasicBlock *BB = AllocBlock.getUniqueSuccessor();
5027	Value *LocalArrayPtr = nullptr;
5028	Value *ReductionFnVal = nullptr;
5029	Value *SwitchArg = nullptr;
5030	for (Instruction &I : *BB) {
5031	if (CallInst *Call = dyn_cast<CallInst>(Val: &I)) {
5032	if (Call->getCalledFunction() !=
5033	OMPBuilder.getOrCreateRuntimeFunctionPtr(
5034	FnID: RuntimeFunction::OMPRTL___kmpc_reduce))
5035	continue;
5036	LocalArrayPtr = Call->getOperand(i_nocapture: 4);
5037	ReductionFnVal = Call->getOperand(i_nocapture: 5);
5038	SwitchArg = Call;
5039	break;
5040	}
5041	}
5042
5043	// Check that the local array is passed to the function.
5044	ASSERT_NE(LocalArrayPtr, nullptr);
5045	EXPECT_EQ(LocalArrayPtr, LocalArray);
5046
5047	// Find the GEP instructions preceding stores to the local array.
5048	Value *FirstArrayElemPtr = nullptr;
5049	Value *SecondArrayElemPtr = nullptr;
5050	EXPECT_EQ(LocalArray->getNumUses(), 3u);
5051	ASSERT_TRUE(
5052	findGEPZeroOne(LocalArray, FirstArrayElemPtr, SecondArrayElemPtr));
5053
5054	// Check that the values stored into the local array are privatized reduction
5055	// variables.
5056	auto *FirstPrivatized = dyn_cast_or_null<AllocaInst>(
5057	Val: findStoredValue<GetElementPtrInst>(AllocaValue: FirstArrayElemPtr));
5058	auto *SecondPrivatized = dyn_cast_or_null<AllocaInst>(
5059	Val: findStoredValue<GetElementPtrInst>(AllocaValue: SecondArrayElemPtr));
5060	ASSERT_NE(FirstPrivatized, nullptr);
5061	ASSERT_NE(SecondPrivatized, nullptr);
5062	ASSERT_TRUE(isa<Instruction>(FirstArrayElemPtr));
5063	EXPECT_TRUE(isSimpleBinaryReduction(
5064	FirstPrivatized, cast<Instruction>(FirstArrayElemPtr)->getParent()));
5065	EXPECT_TRUE(isSimpleBinaryReduction(
5066	SecondPrivatized, cast<Instruction>(FirstArrayElemPtr)->getParent()));
5067
5068	// Check that the result of the runtime reduction call is used for further
5069	// dispatch.
5070	ASSERT_EQ(SwitchArg->getNumUses(), 1u);
5071	SwitchInst *Switch = dyn_cast<SwitchInst>(Val: *SwitchArg->user_begin());
5072	ASSERT_NE(Switch, nullptr);
5073	EXPECT_EQ(Switch->getNumSuccessors(), 3u);
5074	BasicBlock *NonAtomicBB = Switch->case_begin()->getCaseSuccessor();
5075	BasicBlock *AtomicBB = std::next(x: Switch->case_begin())->getCaseSuccessor();
5076
5077	// Non-atomic block contains reductions to the global reduction variable,
5078	// which is passed into the outlined function as an argument.
5079	Value *FirstLoad =
5080	findSingleUserInBlock<LoadInst>(V: FirstPrivatized, BB: NonAtomicBB);
5081	Value *SecondLoad =
5082	findSingleUserInBlock<LoadInst>(V: SecondPrivatized, BB: NonAtomicBB);
5083	EXPECT_TRUE(isValueReducedToFuncArg(FirstLoad, NonAtomicBB));
5084	EXPECT_TRUE(isValueReducedToFuncArg(SecondLoad, NonAtomicBB));
5085
5086	// Atomic block also constains reductions to the global reduction variable.
5087	FirstLoad = findSingleUserInBlock<LoadInst>(V: FirstPrivatized, BB: AtomicBB);
5088	SecondLoad = findSingleUserInBlock<LoadInst>(V: SecondPrivatized, BB: AtomicBB);
5089	auto *FirstAtomic = findSingleUserInBlock<AtomicRMWInst>(V: FirstLoad, BB: AtomicBB);
5090	auto *SecondAtomic =
5091	findSingleUserInBlock<AtomicRMWInst>(V: SecondLoad, BB: AtomicBB);
5092	ASSERT_NE(FirstAtomic, nullptr);
5093	Value *AtomicStorePointer = FirstAtomic->getPointerOperand();
5094	EXPECT_TRUE(isa<Argument>(findAggregateFromValue(AtomicStorePointer)));
5095	ASSERT_NE(SecondAtomic, nullptr);
5096	AtomicStorePointer = SecondAtomic->getPointerOperand();
5097	EXPECT_TRUE(isa<Argument>(findAggregateFromValue(AtomicStorePointer)));
5098
5099	// Check that the separate reduction function also performs (non-atomic)
5100	// reductions after extracting reduction variables from its arguments.
5101	Function *ReductionFn = cast<Function>(Val: ReductionFnVal);
5102	BasicBlock *FnReductionBB = &ReductionFn->getEntryBlock();
5103	Value *FirstLHSPtr;
5104	Value *SecondLHSPtr;
5105	ASSERT_TRUE(
5106	findGEPZeroOne(ReductionFn->getArg(0), FirstLHSPtr, SecondLHSPtr));
5107	Value *Opaque = findSingleUserInBlock<LoadInst>(V: FirstLHSPtr, BB: FnReductionBB);
5108	ASSERT_NE(Opaque, nullptr);
5109	EXPECT_TRUE(isSimpleBinaryReduction(Opaque, FnReductionBB));
5110	Opaque = findSingleUserInBlock<LoadInst>(V: SecondLHSPtr, BB: FnReductionBB);
5111	ASSERT_NE(Opaque, nullptr);
5112	EXPECT_TRUE(isSimpleBinaryReduction(Opaque, FnReductionBB));
5113
5114	Value *FirstRHS;
5115	Value *SecondRHS;
5116	EXPECT_TRUE(findGEPZeroOne(ReductionFn->getArg(1), FirstRHS, SecondRHS));
5117	}
5118
5119	TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
5120	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5121	OpenMPIRBuilder OMPBuilder(*M);
5122	OMPBuilder.Config.IsTargetDevice = false;
5123	OMPBuilder.initialize();
5124	F->setName("func");
5125	IRBuilder<> Builder(BB);
5126
5127	BasicBlock *EnterBB = BasicBlock::Create(Context&: Ctx, Name: "parallel.enter", Parent: F);
5128	Builder.CreateBr(Dest: EnterBB);
5129	Builder.SetInsertPoint(EnterBB);
5130	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5131
5132	// Create variables to be reduced.
5133	InsertPointTy OuterAllocaIP(&F->getEntryBlock(),
5134	F->getEntryBlock().getFirstInsertionPt());
5135	Type *SumType = Builder.getFloatTy();
5136	Type *XorType = Builder.getInt32Ty();
5137	Value *SumReduced;
5138	Value *XorReduced;
5139	{
5140	IRBuilderBase::InsertPointGuard Guard(Builder);
5141	Builder.restoreIP(IP: OuterAllocaIP);
5142	SumReduced = Builder.CreateAlloca(Ty: SumType);
5143	XorReduced = Builder.CreateAlloca(Ty: XorType);
5144	}
5145
5146	// Store initial values of reductions into global variables.
5147	Builder.CreateStore(Val: ConstantFP::get(Ty: Builder.getFloatTy(), V: 0.0), Ptr: SumReduced);
5148	Builder.CreateStore(Val: Builder.getInt32(C: 1), Ptr: XorReduced);
5149
5150	InsertPointTy FirstBodyIP, FirstBodyAllocaIP;
5151	auto FirstBodyGenCB = [&](InsertPointTy InnerAllocaIP,
5152	InsertPointTy CodeGenIP) {
5153	IRBuilderBase::InsertPointGuard Guard(Builder);
5154	Builder.restoreIP(IP: CodeGenIP);
5155
5156	uint32_t StrSize;
5157	Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, SrcLocStrSize&: StrSize);
5158	Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize: StrSize);
5159	Value *TID = OMPBuilder.getOrCreateThreadID(Ident);
5160	Value *SumLocal =
5161	Builder.CreateUIToFP(V: TID, DestTy: Builder.getFloatTy(), Name: "sum.local");
5162	Value *SumPartial = Builder.CreateLoad(Ty: SumType, Ptr: SumReduced, Name: "sum.partial");
5163	Value *Sum = Builder.CreateFAdd(L: SumPartial, R: SumLocal, Name: "sum");
5164	Builder.CreateStore(Val: Sum, Ptr: SumReduced);
5165
5166	FirstBodyIP = Builder.saveIP();
5167	FirstBodyAllocaIP = InnerAllocaIP;
5168	};
5169
5170	InsertPointTy SecondBodyIP, SecondBodyAllocaIP;
5171	auto SecondBodyGenCB = [&](InsertPointTy InnerAllocaIP,
5172	InsertPointTy CodeGenIP) {
5173	IRBuilderBase::InsertPointGuard Guard(Builder);
5174	Builder.restoreIP(IP: CodeGenIP);
5175
5176	uint32_t StrSize;
5177	Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, SrcLocStrSize&: StrSize);
5178	Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize: StrSize);
5179	Value *TID = OMPBuilder.getOrCreateThreadID(Ident);
5180	Value *XorPartial = Builder.CreateLoad(Ty: XorType, Ptr: XorReduced, Name: "xor.partial");
5181	Value *Xor = Builder.CreateXor(LHS: XorPartial, RHS: TID, Name: "xor");
5182	Builder.CreateStore(Val: Xor, Ptr: XorReduced);
5183
5184	SecondBodyIP = Builder.saveIP();
5185	SecondBodyAllocaIP = InnerAllocaIP;
5186	};
5187
5188	// Privatization for reduction creates local copies of reduction variables and
5189	// initializes them to reduction-neutral values. The same privatization
5190	// callback is used for both loops, with dispatch based on the value being
5191	// privatized.
5192	Value *SumPrivatized;
5193	Value *XorPrivatized;
5194	auto PrivCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP,
5195	Value &Original, Value &Inner, Value *&ReplVal) {
5196	IRBuilderBase::InsertPointGuard Guard(Builder);
5197	Builder.restoreIP(IP: InnerAllocaIP);
5198	if (&Original == SumReduced) {
5199	SumPrivatized = Builder.CreateAlloca(Ty: Builder.getFloatTy());
5200	ReplVal = SumPrivatized;
5201	} else if (&Original == XorReduced) {
5202	XorPrivatized = Builder.CreateAlloca(Ty: Builder.getInt32Ty());
5203	ReplVal = XorPrivatized;
5204	} else {
5205	ReplVal = &Inner;
5206	return CodeGenIP;
5207	}
5208
5209	Builder.restoreIP(IP: CodeGenIP);
5210	if (&Original == SumReduced)
5211	Builder.CreateStore(Val: ConstantFP::get(Ty: Builder.getFloatTy(), V: 0.0),
5212	Ptr: SumPrivatized);
5213	else if (&Original == XorReduced)
5214	Builder.CreateStore(Val: Builder.getInt32(C: 0), Ptr: XorPrivatized);
5215
5216	return Builder.saveIP();
5217	};
5218
5219	// Do nothing in finalization.
5220	auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; };
5221
5222	Builder.restoreIP(
5223	OMPBuilder.createParallel(Loc, OuterAllocaIP, FirstBodyGenCB, PrivCB,
5224	FiniCB, /* IfCondition */ nullptr,
5225	/* NumThreads */ nullptr, OMP_PROC_BIND_default,
5226	/* IsCancellable */ false));
5227	InsertPointTy AfterIP = OMPBuilder.createParallel(
5228	{Builder.saveIP(), DL}, OuterAllocaIP, SecondBodyGenCB, PrivCB, FiniCB,
5229	/* IfCondition */ nullptr,
5230	/* NumThreads */ nullptr, OMP_PROC_BIND_default,
5231	/* IsCancellable */ false);
5232
5233	OMPBuilder.createReductions(
5234	Loc: FirstBodyIP, AllocaIP: FirstBodyAllocaIP,
5235	ReductionInfos: {{SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction}});
5236	OMPBuilder.createReductions(
5237	Loc: SecondBodyIP, AllocaIP: SecondBodyAllocaIP,
5238	ReductionInfos: {{XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}});
5239
5240	Builder.restoreIP(IP: AfterIP);
5241	Builder.CreateRetVoid();
5242
5243	OMPBuilder.finalize(Fn: F);
5244
5245	// The IR must be valid.
5246	EXPECT_FALSE(verifyModule(*M));
5247
5248	// Two different outlined functions must have been created.
5249	SmallVector<CallInst *> ForkCalls;
5250	findCalls(F, FnID: omp::RuntimeFunction::OMPRTL___kmpc_fork_call, OMPBuilder,
5251	Calls&: ForkCalls);
5252	ASSERT_EQ(ForkCalls.size(), 2u);
5253	Value *CalleeVal = ForkCalls[0]->getOperand(i_nocapture: 2);
5254	Function *FirstCallee = cast<Function>(Val: CalleeVal);
5255	CalleeVal = ForkCalls[1]->getOperand(i_nocapture: 2);
5256	Function *SecondCallee = cast<Function>(Val: CalleeVal);
5257	EXPECT_NE(FirstCallee, SecondCallee);
5258
5259	// Two different reduction functions must have been created.
5260	SmallVector<CallInst *> ReduceCalls;
5261	findCalls(F: FirstCallee, FnID: omp::RuntimeFunction::OMPRTL___kmpc_reduce, OMPBuilder,
5262	Calls&: ReduceCalls);
5263	ASSERT_EQ(ReduceCalls.size(), 1u);
5264	auto *AddReduction = cast<Function>(Val: ReduceCalls[0]->getOperand(i_nocapture: 5));
5265	ReduceCalls.clear();
5266	findCalls(F: SecondCallee, FnID: omp::RuntimeFunction::OMPRTL___kmpc_reduce,
5267	OMPBuilder, Calls&: ReduceCalls);
5268	auto *XorReduction = cast<Function>(Val: ReduceCalls[0]->getOperand(i_nocapture: 5));
5269	EXPECT_NE(AddReduction, XorReduction);
5270
5271	// Each reduction function does its own kind of reduction.
5272	BasicBlock *FnReductionBB = &AddReduction->getEntryBlock();
5273	Value *FirstLHSPtr = findSingleUserInBlock<GetElementPtrInst>(
5274	V: AddReduction->getArg(i: 0), BB: FnReductionBB);
5275	ASSERT_NE(FirstLHSPtr, nullptr);
5276	Value *Opaque = findSingleUserInBlock<LoadInst>(V: FirstLHSPtr, BB: FnReductionBB);
5277	ASSERT_NE(Opaque, nullptr);
5278	Instruction::BinaryOps Opcode = Instruction::FAdd;
5279	EXPECT_TRUE(isSimpleBinaryReduction(Opaque, FnReductionBB, &Opcode));
5280
5281	FnReductionBB = &XorReduction->getEntryBlock();
5282	Value *SecondLHSPtr = findSingleUserInBlock<GetElementPtrInst>(
5283	V: XorReduction->getArg(i: 0), BB: FnReductionBB);
5284	ASSERT_NE(FirstLHSPtr, nullptr);
5285	Opaque = findSingleUserInBlock<LoadInst>(V: SecondLHSPtr, BB: FnReductionBB);
5286	ASSERT_NE(Opaque, nullptr);
5287	Opcode = Instruction::Xor;
5288	EXPECT_TRUE(isSimpleBinaryReduction(Opaque, FnReductionBB, &Opcode));
5289	}
5290
5291	TEST_F(OpenMPIRBuilderTest, CreateSectionsSimple) {
5292	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5293	using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
5294	OpenMPIRBuilder OMPBuilder(*M);
5295	OMPBuilder.initialize();
5296	F->setName("func");
5297	IRBuilder<> Builder(BB);
5298
5299	BasicBlock *EnterBB = BasicBlock::Create(Context&: Ctx, Name: "sections.enter", Parent: F);
5300	Builder.CreateBr(Dest: EnterBB);
5301	Builder.SetInsertPoint(EnterBB);
5302	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5303
5304	llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
5305	llvm::SmallVector<BasicBlock *, 4> CaseBBs;
5306
5307	auto FiniCB = [&](InsertPointTy IP) {};
5308	auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
5309	SectionCBVector.push_back(Elt: SectionCB);
5310
5311	auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5312	llvm::Value &, llvm::Value &Val,
5313	llvm::Value *&ReplVal) { return CodeGenIP; };
5314	IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
5315	F->getEntryBlock().getFirstInsertionPt());
5316	Builder.restoreIP(IP: OMPBuilder.createSections(Loc, AllocaIP, SectionCBs: SectionCBVector,
5317	PrivCB, FiniCB, IsCancellable: false, IsNowait: false));
5318	Builder.CreateRetVoid(); // Required at the end of the function
5319	EXPECT_NE(F->getEntryBlock().getTerminator(), nullptr);
5320	EXPECT_FALSE(verifyModule(*M, &errs()));
5321	}
5322
5323	TEST_F(OpenMPIRBuilderTest, CreateSections) {
5324	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5325	using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
5326	OpenMPIRBuilder OMPBuilder(*M);
5327	OMPBuilder.initialize();
5328	F->setName("func");
5329	IRBuilder<> Builder(BB);
5330
5331	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5332	llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
5333	llvm::SmallVector<BasicBlock *, 4> CaseBBs;
5334
5335	BasicBlock *SwitchBB = nullptr;
5336	AllocaInst *PrivAI = nullptr;
5337	SwitchInst *Switch = nullptr;
5338
5339	unsigned NumBodiesGenerated = 0;
5340	unsigned NumFiniCBCalls = 0;
5341	PrivAI = Builder.CreateAlloca(Ty: F->arg_begin()->getType());
5342
5343	auto FiniCB = [&](InsertPointTy IP) {
5344	++NumFiniCBCalls;
5345	BasicBlock *IPBB = IP.getBlock();
5346	EXPECT_NE(IPBB->end(), IP.getPoint());
5347	};
5348
5349	auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
5350	++NumBodiesGenerated;
5351	CaseBBs.push_back(Elt: CodeGenIP.getBlock());
5352	SwitchBB = CodeGenIP.getBlock()->getSinglePredecessor();
5353	Builder.restoreIP(IP: CodeGenIP);
5354	Builder.CreateStore(Val: F->arg_begin(), Ptr: PrivAI);
5355	Value *PrivLoad =
5356	Builder.CreateLoad(Ty: F->arg_begin()->getType(), Ptr: PrivAI, Name: "local.alloca");
5357	Builder.CreateICmpNE(LHS: F->arg_begin(), RHS: PrivLoad);
5358	};
5359	auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5360	llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
5361	// TODO: Privatization not implemented yet
5362	return CodeGenIP;
5363	};
5364
5365	SectionCBVector.push_back(Elt: SectionCB);
5366	SectionCBVector.push_back(Elt: SectionCB);
5367
5368	IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
5369	F->getEntryBlock().getFirstInsertionPt());
5370	Builder.restoreIP(IP: OMPBuilder.createSections(Loc, AllocaIP, SectionCBs: SectionCBVector,
5371	PrivCB, FiniCB, IsCancellable: false, IsNowait: false));
5372	Builder.CreateRetVoid(); // Required at the end of the function
5373
5374	// Switch BB's predecessor is loop condition BB, whose successor at index 1 is
5375	// loop's exit BB
5376	BasicBlock *ForExitBB =
5377	SwitchBB->getSinglePredecessor()->getTerminator()->getSuccessor(Idx: 1);
5378	EXPECT_NE(ForExitBB, nullptr);
5379
5380	EXPECT_NE(PrivAI, nullptr);
5381	Function *OutlinedFn = PrivAI->getFunction();
5382	EXPECT_EQ(F, OutlinedFn);
5383	EXPECT_FALSE(verifyModule(*M, &errs()));
5384	EXPECT_EQ(OutlinedFn->arg_size(), 1U);
5385
5386	BasicBlock *LoopPreheaderBB =
5387	OutlinedFn->getEntryBlock().getSingleSuccessor();
5388	// loop variables are 5 - lower bound, upper bound, stride, islastiter, and
5389	// iterator/counter
5390	bool FoundForInit = false;
5391	for (Instruction &Inst : *LoopPreheaderBB) {
5392	if (isa<CallInst>(Val: Inst)) {
5393	if (cast<CallInst>(Val: &Inst)->getCalledFunction()->getName() ==
5394	"__kmpc_for_static_init_4u") {
5395	FoundForInit = true;
5396	}
5397	}
5398	}
5399	EXPECT_EQ(FoundForInit, true);
5400
5401	bool FoundForExit = false;
5402	bool FoundBarrier = false;
5403	for (Instruction &Inst : *ForExitBB) {
5404	if (isa<CallInst>(Val: Inst)) {
5405	if (cast<CallInst>(Val: &Inst)->getCalledFunction()->getName() ==
5406	"__kmpc_for_static_fini") {
5407	FoundForExit = true;
5408	}
5409	if (cast<CallInst>(Val: &Inst)->getCalledFunction()->getName() ==
5410	"__kmpc_barrier") {
5411	FoundBarrier = true;
5412	}
5413	if (FoundForExit && FoundBarrier)
5414	break;
5415	}
5416	}
5417	EXPECT_EQ(FoundForExit, true);
5418	EXPECT_EQ(FoundBarrier, true);
5419
5420	EXPECT_NE(SwitchBB, nullptr);
5421	EXPECT_NE(SwitchBB->getTerminator(), nullptr);
5422	EXPECT_EQ(isa<SwitchInst>(SwitchBB->getTerminator()), true);
5423	Switch = cast<SwitchInst>(Val: SwitchBB->getTerminator());
5424	EXPECT_EQ(Switch->getNumCases(), 2U);
5425
5426	EXPECT_EQ(CaseBBs.size(), 2U);
5427	for (auto *&CaseBB : CaseBBs) {
5428	EXPECT_EQ(CaseBB->getParent(), OutlinedFn);
5429	}
5430
5431	ASSERT_EQ(NumBodiesGenerated, 2U);
5432	ASSERT_EQ(NumFiniCBCalls, 1U);
5433	EXPECT_FALSE(verifyModule(*M, &errs()));
5434	}
5435
5436	TEST_F(OpenMPIRBuilderTest, CreateSectionsNoWait) {
5437	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5438	using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
5439	OpenMPIRBuilder OMPBuilder(*M);
5440	OMPBuilder.initialize();
5441	F->setName("func");
5442	IRBuilder<> Builder(BB);
5443
5444	BasicBlock *EnterBB = BasicBlock::Create(Context&: Ctx, Name: "sections.enter", Parent: F);
5445	Builder.CreateBr(Dest: EnterBB);
5446	Builder.SetInsertPoint(EnterBB);
5447	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5448
5449	IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
5450	F->getEntryBlock().getFirstInsertionPt());
5451	llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
5452	auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5453	llvm::Value &, llvm::Value &Val,
5454	llvm::Value *&ReplVal) { return CodeGenIP; };
5455	auto FiniCB = [&](InsertPointTy IP) {};
5456
5457	Builder.restoreIP(IP: OMPBuilder.createSections(Loc, AllocaIP, SectionCBs: SectionCBVector,
5458	PrivCB, FiniCB, IsCancellable: false, IsNowait: true));
5459	Builder.CreateRetVoid(); // Required at the end of the function
5460	for (auto &Inst : instructions(F&: *F)) {
5461	EXPECT_FALSE(isa<CallInst>(Inst) &&
5462	cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
5463	"__kmpc_barrier" &&
5464	"call to function __kmpc_barrier found with nowait");
5465	}
5466	}
5467
5468	TEST_F(OpenMPIRBuilderTest, CreateOffloadMaptypes) {
5469	OpenMPIRBuilder OMPBuilder(*M);
5470	OMPBuilder.initialize();
5471
5472	IRBuilder<> Builder(BB);
5473
5474	SmallVector<uint64_t> Mappings = {0, 1};
5475	GlobalVariable *OffloadMaptypesGlobal =
5476	OMPBuilder.createOffloadMaptypes(Mappings, VarName: "offload_maptypes");
5477	EXPECT_FALSE(M->global_empty());
5478	EXPECT_EQ(OffloadMaptypesGlobal->getName(), "offload_maptypes");
5479	EXPECT_TRUE(OffloadMaptypesGlobal->isConstant());
5480	EXPECT_TRUE(OffloadMaptypesGlobal->hasGlobalUnnamedAddr());
5481	EXPECT_TRUE(OffloadMaptypesGlobal->hasPrivateLinkage());
5482	EXPECT_TRUE(OffloadMaptypesGlobal->hasInitializer());
5483	Constant *Initializer = OffloadMaptypesGlobal->getInitializer();
5484	EXPECT_TRUE(isa<ConstantDataArray>(Initializer));
5485	ConstantDataArray *MappingInit = dyn_cast<ConstantDataArray>(Val: Initializer);
5486	EXPECT_EQ(MappingInit->getNumElements(), Mappings.size());
5487	EXPECT_TRUE(MappingInit->getType()->getElementType()->isIntegerTy(64));
5488	Constant *CA = ConstantDataArray::get(Context&: Builder.getContext(), Elts&: Mappings);
5489	EXPECT_EQ(MappingInit, CA);
5490	}
5491
5492	TEST_F(OpenMPIRBuilderTest, CreateOffloadMapnames) {
5493	OpenMPIRBuilder OMPBuilder(*M);
5494	OMPBuilder.initialize();
5495
5496	IRBuilder<> Builder(BB);
5497
5498	uint32_t StrSize;
5499	Constant *Cst1 =
5500	OMPBuilder.getOrCreateSrcLocStr(FunctionName: "array1", FileName: "file1", Line: 2, Column: 5, SrcLocStrSize&: StrSize);
5501	Constant *Cst2 =
5502	OMPBuilder.getOrCreateSrcLocStr(FunctionName: "array2", FileName: "file1", Line: 3, Column: 5, SrcLocStrSize&: StrSize);
5503	SmallVector<llvm::Constant *> Names = {Cst1, Cst2};
5504
5505	GlobalVariable *OffloadMaptypesGlobal =
5506	OMPBuilder.createOffloadMapnames(Names, VarName: "offload_mapnames");
5507	EXPECT_FALSE(M->global_empty());
5508	EXPECT_EQ(OffloadMaptypesGlobal->getName(), "offload_mapnames");
5509	EXPECT_TRUE(OffloadMaptypesGlobal->isConstant());
5510	EXPECT_FALSE(OffloadMaptypesGlobal->hasGlobalUnnamedAddr());
5511	EXPECT_TRUE(OffloadMaptypesGlobal->hasPrivateLinkage());
5512	EXPECT_TRUE(OffloadMaptypesGlobal->hasInitializer());
5513	Constant *Initializer = OffloadMaptypesGlobal->getInitializer();
5514	EXPECT_TRUE(isa<Constant>(Initializer->getOperand(0)->stripPointerCasts()));
5515	EXPECT_TRUE(isa<Constant>(Initializer->getOperand(1)->stripPointerCasts()));
5516
5517	GlobalVariable *Name1Gbl =
5518	cast<GlobalVariable>(Val: Initializer->getOperand(i: 0)->stripPointerCasts());
5519	EXPECT_TRUE(isa<ConstantDataArray>(Name1Gbl->getInitializer()));
5520	ConstantDataArray *Name1GblCA =
5521	dyn_cast<ConstantDataArray>(Val: Name1Gbl->getInitializer());
5522	EXPECT_EQ(Name1GblCA->getAsCString(), ";file1;array1;2;5;;");
5523
5524	GlobalVariable *Name2Gbl =
5525	cast<GlobalVariable>(Val: Initializer->getOperand(i: 1)->stripPointerCasts());
5526	EXPECT_TRUE(isa<ConstantDataArray>(Name2Gbl->getInitializer()));
5527	ConstantDataArray *Name2GblCA =
5528	dyn_cast<ConstantDataArray>(Val: Name2Gbl->getInitializer());
5529	EXPECT_EQ(Name2GblCA->getAsCString(), ";file1;array2;3;5;;");
5530
5531	EXPECT_TRUE(Initializer->getType()->getArrayElementType()->isPointerTy());
5532	EXPECT_EQ(Initializer->getType()->getArrayNumElements(), Names.size());
5533	}
5534
5535	TEST_F(OpenMPIRBuilderTest, CreateMapperAllocas) {
5536	OpenMPIRBuilder OMPBuilder(*M);
5537	OMPBuilder.initialize();
5538	F->setName("func");
5539	IRBuilder<> Builder(BB);
5540
5541	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5542
5543	unsigned TotalNbOperand = 2;
5544
5545	OpenMPIRBuilder::MapperAllocas MapperAllocas;
5546	IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
5547	F->getEntryBlock().getFirstInsertionPt());
5548	OMPBuilder.createMapperAllocas(Loc, AllocaIP, NumOperands: TotalNbOperand, MapperAllocas);
5549	EXPECT_NE(MapperAllocas.ArgsBase, nullptr);
5550	EXPECT_NE(MapperAllocas.Args, nullptr);
5551	EXPECT_NE(MapperAllocas.ArgSizes, nullptr);
5552	EXPECT_TRUE(MapperAllocas.ArgsBase->getAllocatedType()->isArrayTy());
5553	ArrayType *ArrType =
5554	dyn_cast<ArrayType>(Val: MapperAllocas.ArgsBase->getAllocatedType());
5555	EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);
5556	EXPECT_TRUE(MapperAllocas.ArgsBase->getAllocatedType()
5557	->getArrayElementType()
5558	->isPointerTy());
5559
5560	EXPECT_TRUE(MapperAllocas.Args->getAllocatedType()->isArrayTy());
5561	ArrType = dyn_cast<ArrayType>(Val: MapperAllocas.Args->getAllocatedType());
5562	EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);
5563	EXPECT_TRUE(MapperAllocas.Args->getAllocatedType()
5564	->getArrayElementType()
5565	->isPointerTy());
5566
5567	EXPECT_TRUE(MapperAllocas.ArgSizes->getAllocatedType()->isArrayTy());
5568	ArrType = dyn_cast<ArrayType>(Val: MapperAllocas.ArgSizes->getAllocatedType());
5569	EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);
5570	EXPECT_TRUE(MapperAllocas.ArgSizes->getAllocatedType()
5571	->getArrayElementType()
5572	->isIntegerTy(64));
5573	}
5574
5575	TEST_F(OpenMPIRBuilderTest, EmitMapperCall) {
5576	OpenMPIRBuilder OMPBuilder(*M);
5577	OMPBuilder.initialize();
5578	F->setName("func");
5579	IRBuilder<> Builder(BB);
5580	LLVMContext &Ctx = M->getContext();
5581
5582	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5583
5584	unsigned TotalNbOperand = 2;
5585
5586	OpenMPIRBuilder::MapperAllocas MapperAllocas;
5587	IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
5588	F->getEntryBlock().getFirstInsertionPt());
5589	OMPBuilder.createMapperAllocas(Loc, AllocaIP, NumOperands: TotalNbOperand, MapperAllocas);
5590
5591	auto *BeginMapperFunc = OMPBuilder.getOrCreateRuntimeFunctionPtr(
5592	FnID: omp::OMPRTL___tgt_target_data_begin_mapper);
5593
5594	SmallVector<uint64_t> Flags = {0, 2};
5595
5596	uint32_t StrSize;
5597	Constant *SrcLocCst =
5598	OMPBuilder.getOrCreateSrcLocStr(FunctionName: "", FileName: "file1", Line: 2, Column: 5, SrcLocStrSize&: StrSize);
5599	Value *SrcLocInfo = OMPBuilder.getOrCreateIdent(SrcLocStr: SrcLocCst, SrcLocStrSize: StrSize);
5600
5601	Constant *Cst1 =
5602	OMPBuilder.getOrCreateSrcLocStr(FunctionName: "array1", FileName: "file1", Line: 2, Column: 5, SrcLocStrSize&: StrSize);
5603	Constant *Cst2 =
5604	OMPBuilder.getOrCreateSrcLocStr(FunctionName: "array2", FileName: "file1", Line: 3, Column: 5, SrcLocStrSize&: StrSize);
5605	SmallVector<llvm::Constant *> Names = {Cst1, Cst2};
5606
5607	GlobalVariable *Maptypes =
5608	OMPBuilder.createOffloadMaptypes(Mappings&: Flags, VarName: ".offload_maptypes");
5609	Value *MaptypesArg = Builder.CreateConstInBoundsGEP2_32(
5610	Ty: ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: TotalNbOperand), Ptr: Maptypes,
5611	/*Idx0=*/0, /*Idx1=*/0);
5612
5613	GlobalVariable *Mapnames =
5614	OMPBuilder.createOffloadMapnames(Names, VarName: ".offload_mapnames");
5615	Value *MapnamesArg = Builder.CreateConstInBoundsGEP2_32(
5616	Ty: ArrayType::get(ElementType: PointerType::getUnqual(C&: Ctx), NumElements: TotalNbOperand), Ptr: Mapnames,
5617	/*Idx0=*/0, /*Idx1=*/0);
5618
5619	OMPBuilder.emitMapperCall(Loc: Builder.saveIP(), MapperFunc: BeginMapperFunc, SrcLocInfo,
5620	MaptypesArg, MapnamesArg, MapperAllocas, DeviceID: -1,
5621	NumOperands: TotalNbOperand);
5622
5623	CallInst *MapperCall = dyn_cast<CallInst>(Val: &BB->back());
5624	EXPECT_NE(MapperCall, nullptr);
5625	EXPECT_EQ(MapperCall->arg_size(), 9U);
5626	EXPECT_EQ(MapperCall->getCalledFunction()->getName(),
5627	"__tgt_target_data_begin_mapper");
5628	EXPECT_EQ(MapperCall->getOperand(0), SrcLocInfo);
5629	EXPECT_TRUE(MapperCall->getOperand(1)->getType()->isIntegerTy(64));
5630	EXPECT_TRUE(MapperCall->getOperand(2)->getType()->isIntegerTy(32));
5631
5632	EXPECT_EQ(MapperCall->getOperand(6), MaptypesArg);
5633	EXPECT_EQ(MapperCall->getOperand(7), MapnamesArg);
5634	EXPECT_TRUE(MapperCall->getOperand(8)->getType()->isPointerTy());
5635	}
5636
5637	TEST_F(OpenMPIRBuilderTest, TargetEnterData) {
5638	OpenMPIRBuilder OMPBuilder(*M);
5639	OMPBuilder.initialize();
5640	F->setName("func");
5641	IRBuilder<> Builder(BB);
5642	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5643
5644	int64_t DeviceID = 2;
5645
5646	AllocaInst *Val1 =
5647	Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: Builder.getInt64(C: 1));
5648	ASSERT_NE(Val1, nullptr);
5649
5650	IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
5651	F->getEntryBlock().getFirstInsertionPt());
5652
5653	llvm::OpenMPIRBuilder::MapInfosTy CombinedInfo;
5654	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5655	auto GenMapInfoCB =
5656	[&](InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
5657	// Get map clause information.
5658	Builder.restoreIP(IP: codeGenIP);
5659
5660	CombinedInfo.BasePointers.emplace_back(Args&: Val1);
5661	CombinedInfo.Pointers.emplace_back(Args&: Val1);
5662	CombinedInfo.DevicePointers.emplace_back(
5663	Args: llvm::OpenMPIRBuilder::DeviceInfoTy::None);
5664	CombinedInfo.Sizes.emplace_back(Args: Builder.getInt64(C: 4));
5665	CombinedInfo.Types.emplace_back(Args: llvm::omp::OpenMPOffloadMappingFlags(1));
5666	uint32_t temp;
5667	CombinedInfo.Names.emplace_back(
5668	Args: OMPBuilder.getOrCreateSrcLocStr(LocStr: "unknown", SrcLocStrSize&: temp));
5669	return CombinedInfo;
5670	};
5671
5672	llvm::OpenMPIRBuilder::TargetDataInfo Info(
5673	/*RequiresDevicePointerInfo=*/false,
5674	/*SeparateBeginEndCalls=*/true);
5675
5676	OMPBuilder.Config.setIsGPU(true);
5677
5678	llvm::omp::RuntimeFunction RTLFunc = OMPRTL___tgt_target_data_begin_mapper;
5679	Builder.restoreIP(IP: OMPBuilder.createTargetData(
5680	Loc, AllocaIP, CodeGenIP: Builder.saveIP(), DeviceID: Builder.getInt64(C: DeviceID),
5681	/* IfCond= */ nullptr, Info, GenMapInfoCB, MapperFunc: &RTLFunc));
5682
5683	CallInst *TargetDataCall = dyn_cast<CallInst>(Val: &BB->back());
5684	EXPECT_NE(TargetDataCall, nullptr);
5685	EXPECT_EQ(TargetDataCall->arg_size(), 9U);
5686	EXPECT_EQ(TargetDataCall->getCalledFunction()->getName(),
5687	"__tgt_target_data_begin_mapper");
5688	EXPECT_TRUE(TargetDataCall->getOperand(1)->getType()->isIntegerTy(64));
5689	EXPECT_TRUE(TargetDataCall->getOperand(2)->getType()->isIntegerTy(32));
5690	EXPECT_TRUE(TargetDataCall->getOperand(8)->getType()->isPointerTy());
5691
5692	Builder.CreateRetVoid();
5693	EXPECT_FALSE(verifyModule(*M, &errs()));
5694	}
5695
5696	TEST_F(OpenMPIRBuilderTest, TargetExitData) {
5697	OpenMPIRBuilder OMPBuilder(*M);
5698	OMPBuilder.initialize();
5699	F->setName("func");
5700	IRBuilder<> Builder(BB);
5701	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5702
5703	int64_t DeviceID = 2;
5704
5705	AllocaInst *Val1 =
5706	Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: Builder.getInt64(C: 1));
5707	ASSERT_NE(Val1, nullptr);
5708
5709	IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
5710	F->getEntryBlock().getFirstInsertionPt());
5711
5712	llvm::OpenMPIRBuilder::MapInfosTy CombinedInfo;
5713	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5714	auto GenMapInfoCB =
5715	[&](InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
5716	// Get map clause information.
5717	Builder.restoreIP(IP: codeGenIP);
5718
5719	CombinedInfo.BasePointers.emplace_back(Args&: Val1);
5720	CombinedInfo.Pointers.emplace_back(Args&: Val1);
5721	CombinedInfo.DevicePointers.emplace_back(
5722	Args: llvm::OpenMPIRBuilder::DeviceInfoTy::None);
5723	CombinedInfo.Sizes.emplace_back(Args: Builder.getInt64(C: 4));
5724	CombinedInfo.Types.emplace_back(Args: llvm::omp::OpenMPOffloadMappingFlags(2));
5725	uint32_t temp;
5726	CombinedInfo.Names.emplace_back(
5727	Args: OMPBuilder.getOrCreateSrcLocStr(LocStr: "unknown", SrcLocStrSize&: temp));
5728	return CombinedInfo;
5729	};
5730
5731	llvm::OpenMPIRBuilder::TargetDataInfo Info(
5732	/*RequiresDevicePointerInfo=*/false,
5733	/*SeparateBeginEndCalls=*/true);
5734
5735	OMPBuilder.Config.setIsGPU(true);
5736
5737	llvm::omp::RuntimeFunction RTLFunc = OMPRTL___tgt_target_data_end_mapper;
5738	Builder.restoreIP(IP: OMPBuilder.createTargetData(
5739	Loc, AllocaIP, CodeGenIP: Builder.saveIP(), DeviceID: Builder.getInt64(C: DeviceID),
5740	/* IfCond= */ nullptr, Info, GenMapInfoCB, MapperFunc: &RTLFunc));
5741
5742	CallInst *TargetDataCall = dyn_cast<CallInst>(Val: &BB->back());
5743	EXPECT_NE(TargetDataCall, nullptr);
5744	EXPECT_EQ(TargetDataCall->arg_size(), 9U);
5745	EXPECT_EQ(TargetDataCall->getCalledFunction()->getName(),
5746	"__tgt_target_data_end_mapper");
5747	EXPECT_TRUE(TargetDataCall->getOperand(1)->getType()->isIntegerTy(64));
5748	EXPECT_TRUE(TargetDataCall->getOperand(2)->getType()->isIntegerTy(32));
5749	EXPECT_TRUE(TargetDataCall->getOperand(8)->getType()->isPointerTy());
5750
5751	Builder.CreateRetVoid();
5752	EXPECT_FALSE(verifyModule(*M, &errs()));
5753	}
5754
5755	TEST_F(OpenMPIRBuilderTest, TargetDataRegion) {
5756	OpenMPIRBuilder OMPBuilder(*M);
5757	OMPBuilder.initialize();
5758	F->setName("func");
5759	IRBuilder<> Builder(BB);
5760	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5761
5762	int64_t DeviceID = 2;
5763
5764	AllocaInst *Val1 =
5765	Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: Builder.getInt64(C: 1));
5766	ASSERT_NE(Val1, nullptr);
5767
5768	AllocaInst *Val2 = Builder.CreateAlloca(Ty: Builder.getPtrTy());
5769	ASSERT_NE(Val2, nullptr);
5770
5771	AllocaInst *Val3 = Builder.CreateAlloca(Ty: Builder.getPtrTy());
5772	ASSERT_NE(Val3, nullptr);
5773
5774	IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
5775	F->getEntryBlock().getFirstInsertionPt());
5776
5777	using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;
5778	llvm::OpenMPIRBuilder::MapInfosTy CombinedInfo;
5779	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5780	auto GenMapInfoCB =
5781	[&](InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
5782	// Get map clause information.
5783	Builder.restoreIP(IP: codeGenIP);
5784	uint32_t temp;
5785
5786	CombinedInfo.BasePointers.emplace_back(Args&: Val1);
5787	CombinedInfo.Pointers.emplace_back(Args&: Val1);
5788	CombinedInfo.DevicePointers.emplace_back(Args: DeviceInfoTy::None);
5789	CombinedInfo.Sizes.emplace_back(Args: Builder.getInt64(C: 4));
5790	CombinedInfo.Types.emplace_back(Args: llvm::omp::OpenMPOffloadMappingFlags(3));
5791	CombinedInfo.Names.emplace_back(
5792	Args: OMPBuilder.getOrCreateSrcLocStr(LocStr: "unknown", SrcLocStrSize&: temp));
5793
5794	CombinedInfo.BasePointers.emplace_back(Args&: Val2);
5795	CombinedInfo.Pointers.emplace_back(Args&: Val2);
5796	CombinedInfo.DevicePointers.emplace_back(Args: DeviceInfoTy::Pointer);
5797	CombinedInfo.Sizes.emplace_back(Args: Builder.getInt64(C: 8));
5798	CombinedInfo.Types.emplace_back(Args: llvm::omp::OpenMPOffloadMappingFlags(67));
5799	CombinedInfo.Names.emplace_back(
5800	Args: OMPBuilder.getOrCreateSrcLocStr(LocStr: "unknown", SrcLocStrSize&: temp));
5801
5802	CombinedInfo.BasePointers.emplace_back(Args&: Val3);
5803	CombinedInfo.Pointers.emplace_back(Args&: Val3);
5804	CombinedInfo.DevicePointers.emplace_back(Args: DeviceInfoTy::Address);
5805	CombinedInfo.Sizes.emplace_back(Args: Builder.getInt64(C: 8));
5806	CombinedInfo.Types.emplace_back(Args: llvm::omp::OpenMPOffloadMappingFlags(67));
5807	CombinedInfo.Names.emplace_back(
5808	Args: OMPBuilder.getOrCreateSrcLocStr(LocStr: "unknown", SrcLocStrSize&: temp));
5809	return CombinedInfo;
5810	};
5811
5812	llvm::OpenMPIRBuilder::TargetDataInfo Info(
5813	/*RequiresDevicePointerInfo=*/true,
5814	/*SeparateBeginEndCalls=*/true);
5815
5816	OMPBuilder.Config.setIsGPU(true);
5817
5818	using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
5819	auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
5820	if (BodyGenType == BodyGenTy::Priv) {
5821	EXPECT_EQ(Info.DevicePtrInfoMap.size(), 2u);
5822	Builder.restoreIP(IP: CodeGenIP);
5823	CallInst *TargetDataCall =
5824	dyn_cast<CallInst>(Val: BB->back().getPrevNode()->getPrevNode());
5825	EXPECT_NE(TargetDataCall, nullptr);
5826	EXPECT_EQ(TargetDataCall->arg_size(), 9U);
5827	EXPECT_EQ(TargetDataCall->getCalledFunction()->getName(),
5828	"__tgt_target_data_begin_mapper");
5829	EXPECT_TRUE(TargetDataCall->getOperand(1)->getType()->isIntegerTy(64));
5830	EXPECT_TRUE(TargetDataCall->getOperand(2)->getType()->isIntegerTy(32));
5831	EXPECT_TRUE(TargetDataCall->getOperand(8)->getType()->isPointerTy());
5832
5833	LoadInst *LI = dyn_cast<LoadInst>(Val: BB->back().getPrevNode());
5834	EXPECT_NE(LI, nullptr);
5835	StoreInst *SI = dyn_cast<StoreInst>(Val: &BB->back());
5836	EXPECT_NE(SI, nullptr);
5837	EXPECT_EQ(SI->getValueOperand(), LI);
5838	EXPECT_EQ(SI->getPointerOperand(), Info.DevicePtrInfoMap[Val2].second);
5839	EXPECT_TRUE(isa<AllocaInst>(Info.DevicePtrInfoMap[Val2].second));
5840	EXPECT_TRUE(isa<GetElementPtrInst>(Info.DevicePtrInfoMap[Val3].second));
5841	Builder.CreateStore(Val: Builder.getInt32(C: 99), Ptr: Val1);
5842	}
5843	return Builder.saveIP();
5844	};
5845
5846	Builder.restoreIP(IP: OMPBuilder.createTargetData(
5847	Loc, AllocaIP, CodeGenIP: Builder.saveIP(), DeviceID: Builder.getInt64(C: DeviceID),
5848	/* IfCond= */ nullptr, Info, GenMapInfoCB, MapperFunc: nullptr, BodyGenCB: BodyCB));
5849
5850	CallInst *TargetDataCall = dyn_cast<CallInst>(Val: &BB->back());
5851	EXPECT_NE(TargetDataCall, nullptr);
5852	EXPECT_EQ(TargetDataCall->arg_size(), 9U);
5853	EXPECT_EQ(TargetDataCall->getCalledFunction()->getName(),
5854	"__tgt_target_data_end_mapper");
5855	EXPECT_TRUE(TargetDataCall->getOperand(1)->getType()->isIntegerTy(64));
5856	EXPECT_TRUE(TargetDataCall->getOperand(2)->getType()->isIntegerTy(32));
5857	EXPECT_TRUE(TargetDataCall->getOperand(8)->getType()->isPointerTy());
5858
5859	// Check that BodyGenCB is still made when IsTargetDevice is set to true.
5860	OMPBuilder.Config.setIsTargetDevice(true);
5861	bool CheckDevicePassBodyGen = false;
5862	auto BodyTargetCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
5863	CheckDevicePassBodyGen = true;
5864	Builder.restoreIP(IP: CodeGenIP);
5865	CallInst *TargetDataCall =
5866	dyn_cast<CallInst>(Val: BB->back().getPrevNode()->getPrevNode());
5867	// Make sure no begin_mapper call is present for device pass.
5868	EXPECT_EQ(TargetDataCall, nullptr);
5869	return Builder.saveIP();
5870	};
5871	Builder.restoreIP(IP: OMPBuilder.createTargetData(
5872	Loc, AllocaIP, CodeGenIP: Builder.saveIP(), DeviceID: Builder.getInt64(C: DeviceID),
5873	/* IfCond= */ nullptr, Info, GenMapInfoCB, MapperFunc: nullptr, BodyGenCB: BodyTargetCB));
5874	EXPECT_TRUE(CheckDevicePassBodyGen);
5875
5876	Builder.CreateRetVoid();
5877	EXPECT_FALSE(verifyModule(*M, &errs()));
5878	}
5879
5880	namespace {
5881	// Some basic handling of argument mapping for the moment
5882	void CreateDefaultMapInfos(llvm::OpenMPIRBuilder &OmpBuilder,
5883	llvm::SmallVectorImpl<llvm::Value *> &Args,
5884	llvm::OpenMPIRBuilder::MapInfosTy &CombinedInfo) {
5885	for (auto Arg : Args) {
5886	CombinedInfo.BasePointers.emplace_back(Args&: Arg);
5887	CombinedInfo.Pointers.emplace_back(Args&: Arg);
5888	uint32_t SrcLocStrSize;
5889	CombinedInfo.Names.emplace_back(Args: OmpBuilder.getOrCreateSrcLocStr(
5890	LocStr: "Unknown loc - stub implementation", SrcLocStrSize));
5891	CombinedInfo.Types.emplace_back(Args: llvm::omp::OpenMPOffloadMappingFlags(
5892	llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
5893	llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM |
5894	llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM));
5895	CombinedInfo.Sizes.emplace_back(Args: OmpBuilder.Builder.getInt64(
5896	C: OmpBuilder.M.getDataLayout().getTypeAllocSize(Ty: Arg->getType())));
5897	}
5898	}
5899	} // namespace
5900
5901	TEST_F(OpenMPIRBuilderTest, TargetRegion) {
5902	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5903	OpenMPIRBuilder OMPBuilder(*M);
5904	OMPBuilder.initialize();
5905	OpenMPIRBuilderConfig Config(false, false, false, false, false, false, false);
5906	OMPBuilder.setConfig(Config);
5907	F->setName("func");
5908	IRBuilder<> Builder(BB);
5909	auto Int32Ty = Builder.getInt32Ty();
5910
5911	AllocaInst *APtr = Builder.CreateAlloca(Ty: Int32Ty, ArraySize: nullptr, Name: "a_ptr");
5912	AllocaInst *BPtr = Builder.CreateAlloca(Ty: Int32Ty, ArraySize: nullptr, Name: "b_ptr");
5913	AllocaInst *CPtr = Builder.CreateAlloca(Ty: Int32Ty, ArraySize: nullptr, Name: "c_ptr");
5914
5915	Builder.CreateStore(Val: Builder.getInt32(C: 10), Ptr: APtr);
5916	Builder.CreateStore(Val: Builder.getInt32(C: 20), Ptr: BPtr);
5917	auto BodyGenCB = [&](InsertPointTy AllocaIP,
5918	InsertPointTy CodeGenIP) -> InsertPointTy {
5919	Builder.restoreIP(IP: CodeGenIP);
5920	LoadInst *AVal = Builder.CreateLoad(Ty: Int32Ty, Ptr: APtr);
5921	LoadInst *BVal = Builder.CreateLoad(Ty: Int32Ty, Ptr: BPtr);
5922	Value *Sum = Builder.CreateAdd(LHS: AVal, RHS: BVal);
5923	Builder.CreateStore(Val: Sum, Ptr: CPtr);
5924	return Builder.saveIP();
5925	};
5926
5927	llvm::SmallVector<llvm::Value *> Inputs;
5928	Inputs.push_back(Elt: APtr);
5929	Inputs.push_back(Elt: BPtr);
5930	Inputs.push_back(Elt: CPtr);
5931
5932	auto SimpleArgAccessorCB =
5933	[&](llvm::Argument &Arg, llvm::Value *Input, llvm::Value *&RetVal,
5934	llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
5935	llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP) {
5936	if (!OMPBuilder.Config.isTargetDevice()) {
5937	RetVal = cast<llvm::Value>(Val: &Arg);
5938	return CodeGenIP;
5939	}
5940
5941	Builder.restoreIP(IP: AllocaIP);
5942
5943	llvm::Value *Addr = Builder.CreateAlloca(
5944	Ty: Arg.getType()->isPointerTy()
5945	? Arg.getType()
5946	: Type::getInt64Ty(C&: Builder.getContext()),
5947	AddrSpace: OMPBuilder.M.getDataLayout().getAllocaAddrSpace());
5948	llvm::Value *AddrAscast =
5949	Builder.CreatePointerBitCastOrAddrSpaceCast(V: Addr, DestTy: Input->getType());
5950	Builder.CreateStore(Val: &Arg, Ptr: AddrAscast);
5951
5952	Builder.restoreIP(IP: CodeGenIP);
5953
5954	RetVal = Builder.CreateLoad(Ty: Arg.getType(), Ptr: AddrAscast);
5955
5956	return Builder.saveIP();
5957	};
5958
5959	llvm::OpenMPIRBuilder::MapInfosTy CombinedInfos;
5960	auto GenMapInfoCB = [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP)
5961	-> llvm::OpenMPIRBuilder::MapInfosTy & {
5962	CreateDefaultMapInfos(OmpBuilder&: OMPBuilder, Args&: Inputs, CombinedInfo&: CombinedInfos);
5963	return CombinedInfos;
5964	};
5965
5966	TargetRegionEntryInfo EntryInfo("func", 42, 4711, 17);
5967	OpenMPIRBuilder::LocationDescription OmpLoc({Builder.saveIP(), DL});
5968	Builder.restoreIP(IP: OMPBuilder.createTarget(
5969	Loc: OmpLoc, AllocaIP: Builder.saveIP(), CodeGenIP: Builder.saveIP(), EntryInfo, NumTeams: -1, NumThreads: 0, Inputs,
5970	GenMapInfoCB, BodyGenCB, ArgAccessorFuncCB: SimpleArgAccessorCB));
5971	OMPBuilder.finalize();
5972	Builder.CreateRetVoid();
5973
5974	// Check the kernel launch sequence
5975	auto Iter = F->getEntryBlock().rbegin();
5976	EXPECT_TRUE(isa<BranchInst>(&*(Iter)));
5977	BranchInst *Branch = dyn_cast<BranchInst>(Val: &*(Iter));
5978	EXPECT_TRUE(isa<CmpInst>(&*(++Iter)));
5979	EXPECT_TRUE(isa<CallInst>(&*(++Iter)));
5980	CallInst *Call = dyn_cast<CallInst>(Val: &*(Iter));
5981
5982	// Check that the kernel launch function is called
5983	Function *KernelLaunchFunc = Call->getCalledFunction();
5984	EXPECT_NE(KernelLaunchFunc, nullptr);
5985	StringRef FunctionName = KernelLaunchFunc->getName();
5986	EXPECT_TRUE(FunctionName.starts_with("__tgt_target_kernel"));
5987
5988	// Check the fallback call
5989	BasicBlock *FallbackBlock = Branch->getSuccessor(i: 0);
5990	Iter = FallbackBlock->rbegin();
5991	CallInst *FCall = dyn_cast<CallInst>(Val: &*(++Iter));
5992	EXPECT_NE(FCall, nullptr);
5993
5994	// Check that the correct aguments are passed in
5995	for (auto ArgInput : zip(t: FCall->args(), u&: Inputs)) {
5996	EXPECT_EQ(std::get<0>(ArgInput), std::get<1>(ArgInput));
5997	}
5998
5999	// Check that the outlined function exists with the expected prefix
6000	Function *OutlinedFunc = FCall->getCalledFunction();
6001	EXPECT_NE(OutlinedFunc, nullptr);
6002	StringRef FunctionName2 = OutlinedFunc->getName();
6003	EXPECT_TRUE(FunctionName2.starts_with("__omp_offloading"));
6004
6005	EXPECT_FALSE(verifyModule(*M, &errs()));
6006	}
6007
6008	TEST_F(OpenMPIRBuilderTest, TargetRegionDevice) {
6009	OpenMPIRBuilder OMPBuilder(*M);
6010	OMPBuilder.setConfig(
6011	OpenMPIRBuilderConfig(true, false, false, false, false, false, false));
6012	OMPBuilder.initialize();
6013
6014	F->setName("func");
6015	IRBuilder<> Builder(BB);
6016	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
6017
6018	LoadInst *Value = nullptr;
6019	StoreInst *TargetStore = nullptr;
6020	llvm::SmallVector<llvm::Value *, 2> CapturedArgs = {
6021	Constant::getNullValue(Ty: PointerType::get(C&: Ctx, AddressSpace: 0)),
6022	Constant::getNullValue(Ty: PointerType::get(C&: Ctx, AddressSpace: 0))};
6023
6024	auto SimpleArgAccessorCB =
6025	[&](llvm::Argument &Arg, llvm::Value *Input, llvm::Value *&RetVal,
6026	llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
6027	llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP) {
6028	if (!OMPBuilder.Config.isTargetDevice()) {
6029	RetVal = cast<llvm::Value>(Val: &Arg);
6030	return CodeGenIP;
6031	}
6032
6033	Builder.restoreIP(IP: AllocaIP);
6034
6035	llvm::Value *Addr = Builder.CreateAlloca(
6036	Ty: Arg.getType()->isPointerTy()
6037	? Arg.getType()
6038	: Type::getInt64Ty(C&: Builder.getContext()),
6039	AddrSpace: OMPBuilder.M.getDataLayout().getAllocaAddrSpace());
6040	llvm::Value *AddrAscast =
6041	Builder.CreatePointerBitCastOrAddrSpaceCast(V: Addr, DestTy: Input->getType());
6042	Builder.CreateStore(Val: &Arg, Ptr: AddrAscast);
6043
6044	Builder.restoreIP(IP: CodeGenIP);
6045
6046	RetVal = Builder.CreateLoad(Ty: Arg.getType(), Ptr: AddrAscast);
6047
6048	return Builder.saveIP();
6049	};
6050
6051	llvm::OpenMPIRBuilder::MapInfosTy CombinedInfos;
6052	auto GenMapInfoCB = [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP)
6053	-> llvm::OpenMPIRBuilder::MapInfosTy & {
6054	CreateDefaultMapInfos(OmpBuilder&: OMPBuilder, Args&: CapturedArgs, CombinedInfo&: CombinedInfos);
6055	return CombinedInfos;
6056	};
6057
6058	auto BodyGenCB = [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
6059	OpenMPIRBuilder::InsertPointTy CodeGenIP)
6060	-> OpenMPIRBuilder::InsertPointTy {
6061	Builder.restoreIP(IP: CodeGenIP);
6062	Value = Builder.CreateLoad(Ty: Type::getInt32Ty(C&: Ctx), Ptr: CapturedArgs[0]);
6063	TargetStore = Builder.CreateStore(Val: Value, Ptr: CapturedArgs[1]);
6064	return Builder.saveIP();
6065	};
6066
6067	IRBuilder<>::InsertPoint EntryIP(&F->getEntryBlock(),
6068	F->getEntryBlock().getFirstInsertionPt());
6069	TargetRegionEntryInfo EntryInfo("parent", /*DeviceID=*/1, /*FileID=*/2,
6070	/*Line=*/3, /*Count=*/0);
6071
6072	Builder.restoreIP(
6073	IP: OMPBuilder.createTarget(Loc, AllocaIP: EntryIP, CodeGenIP: EntryIP, EntryInfo, /*NumTeams=*/-1,
6074	/*NumThreads=*/0, Inputs&: CapturedArgs, GenMapInfoCB,
6075	BodyGenCB, ArgAccessorFuncCB: SimpleArgAccessorCB));
6076
6077	Builder.CreateRetVoid();
6078	OMPBuilder.finalize();
6079
6080	// Check outlined function
6081	EXPECT_FALSE(verifyModule(*M, &errs()));
6082	EXPECT_NE(TargetStore, nullptr);
6083	Function *OutlinedFn = TargetStore->getFunction();
6084	EXPECT_NE(F, OutlinedFn);
6085
6086	EXPECT_TRUE(OutlinedFn->hasWeakODRLinkage());
6087	// Account for the "implicit" first argument.
6088	EXPECT_EQ(OutlinedFn->getName(), "__omp_offloading_1_2_parent_l3");
6089	EXPECT_EQ(OutlinedFn->arg_size(), 3U);
6090	EXPECT_TRUE(OutlinedFn->getArg(1)->getType()->isPointerTy());
6091	EXPECT_TRUE(OutlinedFn->getArg(2)->getType()->isPointerTy());
6092
6093	// Check entry block
6094	auto &EntryBlock = OutlinedFn->getEntryBlock();
6095	Instruction *Alloca1 = EntryBlock.getFirstNonPHI();
6096	EXPECT_NE(Alloca1, nullptr);
6097
6098	EXPECT_TRUE(isa<AllocaInst>(Alloca1));
6099	auto *Store1 = Alloca1->getNextNode();
6100	EXPECT_TRUE(isa<StoreInst>(Store1));
6101	auto *Alloca2 = Store1->getNextNode();
6102	EXPECT_TRUE(isa<AllocaInst>(Alloca2));
6103	auto *Store2 = Alloca2->getNextNode();
6104	EXPECT_TRUE(isa<StoreInst>(Store2));
6105
6106	auto *InitCall = dyn_cast<CallInst>(Val: Store2->getNextNode());
6107	EXPECT_NE(InitCall, nullptr);
6108	EXPECT_EQ(InitCall->getCalledFunction()->getName(), "__kmpc_target_init");
6109	EXPECT_EQ(InitCall->arg_size(), 2U);
6110	EXPECT_TRUE(isa<GlobalVariable>(InitCall->getArgOperand(0)));
6111	auto *KernelEnvGV = cast<GlobalVariable>(Val: InitCall->getArgOperand(i: 0));
6112	EXPECT_TRUE(isa<ConstantStruct>(KernelEnvGV->getInitializer()));
6113	auto *KernelEnvC = cast<ConstantStruct>(Val: KernelEnvGV->getInitializer());
6114	EXPECT_TRUE(isa<ConstantStruct>(KernelEnvC->getAggregateElement(0U)));
6115	auto ConfigC = cast<ConstantStruct>(Val: KernelEnvC->getAggregateElement(Elt: 0U));
6116	EXPECT_EQ(ConfigC->getAggregateElement(0U),
6117	ConstantInt::get(Type::getInt8Ty(Ctx), true));
6118	EXPECT_EQ(ConfigC->getAggregateElement(1U),
6119	ConstantInt::get(Type::getInt8Ty(Ctx), true));
6120	EXPECT_EQ(ConfigC->getAggregateElement(2U),
6121	ConstantInt::get(Type::getInt8Ty(Ctx), OMP_TGT_EXEC_MODE_GENERIC));
6122
6123	auto *EntryBlockBranch = EntryBlock.getTerminator();
6124	EXPECT_NE(EntryBlockBranch, nullptr);
6125	EXPECT_EQ(EntryBlockBranch->getNumSuccessors(), 2U);
6126
6127	// Check user code block
6128	auto *UserCodeBlock = EntryBlockBranch->getSuccessor(Idx: 0);
6129	EXPECT_EQ(UserCodeBlock->getName(), "user_code.entry");
6130	auto *Load1 = UserCodeBlock->getFirstNonPHI();
6131	EXPECT_TRUE(isa<LoadInst>(Load1));
6132	auto *Load2 = Load1->getNextNode();
6133	EXPECT_TRUE(isa<LoadInst>(Load2));
6134
6135	auto *Value1 = Load2->getNextNode();
6136	EXPECT_EQ(Value1, Value);
6137	EXPECT_EQ(Value1->getNextNode(), TargetStore);
6138	auto *Deinit = TargetStore->getNextNode();
6139	EXPECT_NE(Deinit, nullptr);
6140
6141	auto *DeinitCall = dyn_cast<CallInst>(Val: Deinit);
6142	EXPECT_NE(DeinitCall, nullptr);
6143	EXPECT_EQ(DeinitCall->getCalledFunction()->getName(), "__kmpc_target_deinit");
6144	EXPECT_EQ(DeinitCall->arg_size(), 0U);
6145
6146	EXPECT_TRUE(isa<ReturnInst>(DeinitCall->getNextNode()));
6147
6148	// Check exit block
6149	auto *ExitBlock = EntryBlockBranch->getSuccessor(Idx: 1);
6150	EXPECT_EQ(ExitBlock->getName(), "worker.exit");
6151	EXPECT_TRUE(isa<ReturnInst>(ExitBlock->getFirstNonPHI()));
6152	}
6153
6154	TEST_F(OpenMPIRBuilderTest, ConstantAllocaRaise) {
6155	OpenMPIRBuilder OMPBuilder(*M);
6156	OMPBuilder.setConfig(
6157	OpenMPIRBuilderConfig(true, false, false, false, false, false, false));
6158	OMPBuilder.initialize();
6159
6160	F->setName("func");
6161	IRBuilder<> Builder(BB);
6162	OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
6163
6164	LoadInst *Value = nullptr;
6165	StoreInst *TargetStore = nullptr;
6166	llvm::SmallVector<llvm::Value *, 1> CapturedArgs = {
6167	Constant::getNullValue(Ty: PointerType::get(C&: Ctx, AddressSpace: 0))};
6168
6169	auto SimpleArgAccessorCB =
6170	[&](llvm::Argument &Arg, llvm::Value *Input, llvm::Value *&RetVal,
6171	llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
6172	llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP) {
6173	if (!OMPBuilder.Config.isTargetDevice()) {
6174	RetVal = cast<llvm::Value>(Val: &Arg);
6175	return CodeGenIP;
6176	}
6177
6178	Builder.restoreIP(IP: AllocaIP);
6179
6180	llvm::Value *Addr = Builder.CreateAlloca(
6181	Ty: Arg.getType()->isPointerTy()
6182	? Arg.getType()
6183	: Type::getInt64Ty(C&: Builder.getContext()),
6184	AddrSpace: OMPBuilder.M.getDataLayout().getAllocaAddrSpace());
6185	llvm::Value *AddrAscast =
6186	Builder.CreatePointerBitCastOrAddrSpaceCast(V: Addr, DestTy: Input->getType());
6187	Builder.CreateStore(Val: &Arg, Ptr: AddrAscast);
6188
6189	Builder.restoreIP(IP: CodeGenIP);
6190
6191	RetVal = Builder.CreateLoad(Ty: Arg.getType(), Ptr: AddrAscast);
6192
6193	return Builder.saveIP();
6194	};
6195
6196	llvm::OpenMPIRBuilder::MapInfosTy CombinedInfos;
6197	auto GenMapInfoCB = [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP)
6198	-> llvm::OpenMPIRBuilder::MapInfosTy & {
6199	CreateDefaultMapInfos(OmpBuilder&: OMPBuilder, Args&: CapturedArgs, CombinedInfo&: CombinedInfos);
6200	return CombinedInfos;
6201	};
6202
6203	llvm::Value *RaiseAlloca = nullptr;
6204
6205	auto BodyGenCB = [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
6206	OpenMPIRBuilder::InsertPointTy CodeGenIP)
6207	-> OpenMPIRBuilder::InsertPointTy {
6208	Builder.restoreIP(IP: CodeGenIP);
6209	RaiseAlloca = Builder.CreateAlloca(Ty: Builder.getInt32Ty());
6210	Value = Builder.CreateLoad(Ty: Type::getInt32Ty(C&: Ctx), Ptr: CapturedArgs[0]);
6211	TargetStore = Builder.CreateStore(Val: Value, Ptr: RaiseAlloca);
6212	return Builder.saveIP();
6213	};
6214
6215	IRBuilder<>::InsertPoint EntryIP(&F->getEntryBlock(),
6216	F->getEntryBlock().getFirstInsertionPt());
6217	TargetRegionEntryInfo EntryInfo("parent", /*DeviceID=*/1, /*FileID=*/2,
6218	/*Line=*/3, /*Count=*/0);
6219
6220	Builder.restoreIP(
6221	IP: OMPBuilder.createTarget(Loc, AllocaIP: EntryIP, CodeGenIP: EntryIP, EntryInfo, /*NumTeams=*/-1,
6222	/*NumThreads=*/0, Inputs&: CapturedArgs, GenMapInfoCB,
6223	BodyGenCB, ArgAccessorFuncCB: SimpleArgAccessorCB));
6224
6225	Builder.CreateRetVoid();
6226	OMPBuilder.finalize();
6227
6228	// Check outlined function
6229	EXPECT_FALSE(verifyModule(*M, &errs()));
6230	EXPECT_NE(TargetStore, nullptr);
6231	Function *OutlinedFn = TargetStore->getFunction();
6232	EXPECT_NE(F, OutlinedFn);
6233
6234	EXPECT_TRUE(OutlinedFn->hasWeakODRLinkage());
6235	// Account for the "implicit" first argument.
6236	EXPECT_EQ(OutlinedFn->getName(), "__omp_offloading_1_2_parent_l3");
6237	EXPECT_EQ(OutlinedFn->arg_size(), 2U);
6238	EXPECT_TRUE(OutlinedFn->getArg(1)->getType()->isPointerTy());
6239
6240	// Check entry block, to see if we have raised our alloca
6241	// from the body to the entry block.
6242	auto &EntryBlock = OutlinedFn->getEntryBlock();
6243
6244	// Check that we have moved our alloca created in the
6245	// BodyGenCB function, to the top of the function.
6246	Instruction *Alloca1 = EntryBlock.getFirstNonPHI();
6247	EXPECT_NE(Alloca1, nullptr);
6248	EXPECT_TRUE(isa<AllocaInst>(Alloca1));
6249	EXPECT_EQ(Alloca1, RaiseAlloca);
6250
6251	// Verify we have not altered the rest of the function
6252	// inappropriately with our alloca movement.
6253	auto *Alloca2 = Alloca1->getNextNode();
6254	EXPECT_TRUE(isa<AllocaInst>(Alloca2));
6255	auto *Store2 = Alloca2->getNextNode();
6256	EXPECT_TRUE(isa<StoreInst>(Store2));
6257
6258	auto *InitCall = dyn_cast<CallInst>(Val: Store2->getNextNode());
6259	EXPECT_NE(InitCall, nullptr);
6260	EXPECT_EQ(InitCall->getCalledFunction()->getName(), "__kmpc_target_init");
6261	EXPECT_EQ(InitCall->arg_size(), 2U);
6262	EXPECT_TRUE(isa<GlobalVariable>(InitCall->getArgOperand(0)));
6263	auto *KernelEnvGV = cast<GlobalVariable>(Val: InitCall->getArgOperand(i: 0));
6264	EXPECT_TRUE(isa<ConstantStruct>(KernelEnvGV->getInitializer()));
6265	auto *KernelEnvC = cast<ConstantStruct>(Val: KernelEnvGV->getInitializer());
6266	EXPECT_TRUE(isa<ConstantStruct>(KernelEnvC->getAggregateElement(0U)));
6267	auto *ConfigC = cast<ConstantStruct>(Val: KernelEnvC->getAggregateElement(Elt: 0U));
6268	EXPECT_EQ(ConfigC->getAggregateElement(0U),
6269	ConstantInt::get(Type::getInt8Ty(Ctx), true));
6270	EXPECT_EQ(ConfigC->getAggregateElement(1U),
6271	ConstantInt::get(Type::getInt8Ty(Ctx), true));
6272	EXPECT_EQ(ConfigC->getAggregateElement(2U),
6273	ConstantInt::get(Type::getInt8Ty(Ctx), OMP_TGT_EXEC_MODE_GENERIC));
6274
6275	auto *EntryBlockBranch = EntryBlock.getTerminator();
6276	EXPECT_NE(EntryBlockBranch, nullptr);
6277	EXPECT_EQ(EntryBlockBranch->getNumSuccessors(), 2U);
6278
6279	// Check user code block
6280	auto *UserCodeBlock = EntryBlockBranch->getSuccessor(Idx: 0);
6281	EXPECT_EQ(UserCodeBlock->getName(), "user_code.entry");
6282	auto *Load1 = UserCodeBlock->getFirstNonPHI();
6283	EXPECT_TRUE(isa<LoadInst>(Load1));
6284	auto *Load2 = Load1->getNextNode();
6285	EXPECT_TRUE(isa<LoadInst>(Load2));
6286	EXPECT_EQ(Load2, Value);
6287	EXPECT_EQ(Load2->getNextNode(), TargetStore);
6288	auto *Deinit = TargetStore->getNextNode();
6289	EXPECT_NE(Deinit, nullptr);
6290
6291	auto *DeinitCall = dyn_cast<CallInst>(Val: Deinit);
6292	EXPECT_NE(DeinitCall, nullptr);
6293	EXPECT_EQ(DeinitCall->getCalledFunction()->getName(), "__kmpc_target_deinit");
6294	EXPECT_EQ(DeinitCall->arg_size(), 0U);
6295
6296	EXPECT_TRUE(isa<ReturnInst>(DeinitCall->getNextNode()));
6297
6298	// Check exit block
6299	auto *ExitBlock = EntryBlockBranch->getSuccessor(Idx: 1);
6300	EXPECT_EQ(ExitBlock->getName(), "worker.exit");
6301	EXPECT_TRUE(isa<ReturnInst>(ExitBlock->getFirstNonPHI()));
6302	}
6303
6304	TEST_F(OpenMPIRBuilderTest, CreateTask) {
6305	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
6306	OpenMPIRBuilder OMPBuilder(*M);
6307	OMPBuilder.Config.IsTargetDevice = false;
6308	OMPBuilder.initialize();
6309	F->setName("func");
6310	IRBuilder<> Builder(BB);
6311
6312	AllocaInst *ValPtr32 = Builder.CreateAlloca(Ty: Builder.getInt32Ty());
6313	AllocaInst *ValPtr128 = Builder.CreateAlloca(Ty: Builder.getInt128Ty());
6314	Value *Val128 =
6315	Builder.CreateLoad(Ty: Builder.getInt128Ty(), Ptr: ValPtr128, Name: "bodygen.load");
6316
6317	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
6318	Builder.restoreIP(IP: AllocaIP);
6319	AllocaInst *Local128 = Builder.CreateAlloca(Ty: Builder.getInt128Ty(), ArraySize: nullptr,
6320	Name: "bodygen.alloca128");
6321
6322	Builder.restoreIP(IP: CodeGenIP);
6323	// Loading and storing captured pointer and values
6324	Builder.CreateStore(Val: Val128, Ptr: Local128);
6325	Value *Val32 = Builder.CreateLoad(Ty: ValPtr32->getAllocatedType(), Ptr: ValPtr32,
6326	Name: "bodygen.load32");
6327
6328	LoadInst *PrivLoad128 = Builder.CreateLoad(
6329	Ty: Local128->getAllocatedType(), Ptr: Local128, Name: "bodygen.local.load128");
6330	Value *Cmp = Builder.CreateICmpNE(
6331	LHS: Val32, RHS: Builder.CreateTrunc(V: PrivLoad128, DestTy: Val32->getType()));
6332	Instruction *ThenTerm, *ElseTerm;
6333	SplitBlockAndInsertIfThenElse(Cond: Cmp, SplitBefore: CodeGenIP.getBlock()->getTerminator(),
6334	ThenTerm: &ThenTerm, ElseTerm: &ElseTerm);
6335	};
6336
6337	BasicBlock *AllocaBB = Builder.GetInsertBlock();
6338	BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "alloca.split");
6339	OpenMPIRBuilder::LocationDescription Loc(
6340	InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
6341	Builder.restoreIP(IP: OMPBuilder.createTask(
6342	Loc, AllocaIP: InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
6343	BodyGenCB));
6344	OMPBuilder.finalize();
6345	Builder.CreateRetVoid();
6346
6347	EXPECT_FALSE(verifyModule(*M, &errs()));
6348
6349	CallInst *TaskAllocCall = dyn_cast<CallInst>(
6350	Val: OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc)
6351	->user_back());
6352
6353	// Verify the Ident argument
6354	GlobalVariable *Ident = cast<GlobalVariable>(Val: TaskAllocCall->getArgOperand(i: 0));
6355	ASSERT_NE(Ident, nullptr);
6356	EXPECT_TRUE(Ident->hasInitializer());
6357	Constant *Initializer = Ident->getInitializer();
6358	GlobalVariable *SrcStrGlob =
6359	cast<GlobalVariable>(Val: Initializer->getOperand(i: 4)->stripPointerCasts());
6360	ASSERT_NE(SrcStrGlob, nullptr);
6361	ConstantDataArray *SrcSrc =
6362	dyn_cast<ConstantDataArray>(Val: SrcStrGlob->getInitializer());
6363	ASSERT_NE(SrcSrc, nullptr);
6364
6365	// Verify the num_threads argument.
6366	CallInst *GTID = dyn_cast<CallInst>(Val: TaskAllocCall->getArgOperand(i: 1));
6367	ASSERT_NE(GTID, nullptr);
6368	EXPECT_EQ(GTID->arg_size(), 1U);
6369	EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
6370
6371	// Verify the flags
6372	// TODO: Check for others flags. Currently testing only for tiedness.
6373	ConstantInt *Flags = dyn_cast<ConstantInt>(Val: TaskAllocCall->getArgOperand(i: 2));
6374	ASSERT_NE(Flags, nullptr);
6375	EXPECT_EQ(Flags->getSExtValue(), 1);
6376
6377	// Verify the data size
6378	ConstantInt *DataSize =
6379	dyn_cast<ConstantInt>(Val: TaskAllocCall->getArgOperand(i: 3));
6380	ASSERT_NE(DataSize, nullptr);
6381	EXPECT_EQ(DataSize->getSExtValue(), 40);
6382
6383	ConstantInt *SharedsSize =
6384	dyn_cast<ConstantInt>(Val: TaskAllocCall->getOperand(i_nocapture: 4));
6385	EXPECT_EQ(SharedsSize->getSExtValue(),
6386	24); // 64-bit pointer + 128-bit integer
6387
6388	// Verify Wrapper function
6389	Function *OutlinedFn =
6390	dyn_cast<Function>(Val: TaskAllocCall->getArgOperand(i: 5)->stripPointerCasts());
6391	ASSERT_NE(OutlinedFn, nullptr);
6392
6393	LoadInst *SharedsLoad = dyn_cast<LoadInst>(Val: OutlinedFn->begin()->begin());
6394	ASSERT_NE(SharedsLoad, nullptr);
6395	EXPECT_EQ(SharedsLoad->getPointerOperand(), OutlinedFn->getArg(1));
6396
6397	EXPECT_FALSE(OutlinedFn->isDeclaration());
6398	EXPECT_EQ(OutlinedFn->getArg(0)->getType(), Builder.getInt32Ty());
6399
6400	// Verify that the data argument is used only once, and that too in the load
6401	// instruction that is then used for accessing shared data.
6402	Value *DataPtr = OutlinedFn->getArg(i: 1);
6403	EXPECT_EQ(DataPtr->getNumUses(), 1U);
6404	EXPECT_TRUE(isa<LoadInst>(DataPtr->uses().begin()->getUser()));
6405	Value *Data = DataPtr->uses().begin()->getUser();
6406	EXPECT_TRUE(all_of(Data->uses(), [](Use &U) {
6407	return isa<GetElementPtrInst>(U.getUser());
6408	}));
6409
6410	// Verify the presence of `trunc` and `icmp` instructions in Outlined function
6411	EXPECT_TRUE(any_of(instructions(OutlinedFn),
6412	[](Instruction &inst) { return isa<TruncInst>(&inst); }));
6413	EXPECT_TRUE(any_of(instructions(OutlinedFn),
6414	[](Instruction &inst) { return isa<ICmpInst>(&inst); }));
6415
6416	// Verify the execution of the task
6417	CallInst *TaskCall = dyn_cast<CallInst>(
6418	Val: OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task)
6419	->user_back());
6420	ASSERT_NE(TaskCall, nullptr);
6421	EXPECT_EQ(TaskCall->getArgOperand(0), Ident);
6422	EXPECT_EQ(TaskCall->getArgOperand(1), GTID);
6423	EXPECT_EQ(TaskCall->getArgOperand(2), TaskAllocCall);
6424
6425	// Verify that the argument data has been copied
6426	for (User *in : TaskAllocCall->users()) {
6427	if (MemCpyInst *memCpyInst = dyn_cast<MemCpyInst>(Val: in)) {
6428	EXPECT_EQ(memCpyInst->getDest(), TaskAllocCall);
6429	}
6430	}
6431	}
6432
6433	TEST_F(OpenMPIRBuilderTest, CreateTaskNoArgs) {
6434	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
6435	OpenMPIRBuilder OMPBuilder(*M);
6436	OMPBuilder.Config.IsTargetDevice = false;
6437	OMPBuilder.initialize();
6438	F->setName("func");
6439	IRBuilder<> Builder(BB);
6440
6441	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
6442
6443	BasicBlock *AllocaBB = Builder.GetInsertBlock();
6444	BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "alloca.split");
6445	OpenMPIRBuilder::LocationDescription Loc(
6446	InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
6447	Builder.restoreIP(IP: OMPBuilder.createTask(
6448	Loc, AllocaIP: InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
6449	BodyGenCB));
6450	OMPBuilder.finalize();
6451	Builder.CreateRetVoid();
6452
6453	EXPECT_FALSE(verifyModule(*M, &errs()));
6454
6455	// Check that the outlined function has only one argument.
6456	CallInst *TaskAllocCall = dyn_cast<CallInst>(
6457	Val: OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc)
6458	->user_back());
6459	Function *OutlinedFn = dyn_cast<Function>(Val: TaskAllocCall->getArgOperand(i: 5));
6460	ASSERT_NE(OutlinedFn, nullptr);
6461	ASSERT_EQ(OutlinedFn->arg_size(), 1U);
6462	}
6463
6464	TEST_F(OpenMPIRBuilderTest, CreateTaskUntied) {
6465	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
6466	OpenMPIRBuilder OMPBuilder(*M);
6467	OMPBuilder.Config.IsTargetDevice = false;
6468	OMPBuilder.initialize();
6469	F->setName("func");
6470	IRBuilder<> Builder(BB);
6471	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
6472	BasicBlock *AllocaBB = Builder.GetInsertBlock();
6473	BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "alloca.split");
6474	OpenMPIRBuilder::LocationDescription Loc(
6475	InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
6476	Builder.restoreIP(IP: OMPBuilder.createTask(
6477	Loc, AllocaIP: InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB,
6478	/*Tied=*/false));
6479	OMPBuilder.finalize();
6480	Builder.CreateRetVoid();
6481
6482	// Check for the `Tied` argument
6483	CallInst *TaskAllocCall = dyn_cast<CallInst>(
6484	Val: OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc)
6485	->user_back());
6486	ASSERT_NE(TaskAllocCall, nullptr);
6487	ConstantInt *Flags = dyn_cast<ConstantInt>(Val: TaskAllocCall->getArgOperand(i: 2));
6488	ASSERT_NE(Flags, nullptr);
6489	EXPECT_EQ(Flags->getZExtValue() & 1U, 0U);
6490
6491	EXPECT_FALSE(verifyModule(*M, &errs()));
6492	}
6493
6494	TEST_F(OpenMPIRBuilderTest, CreateTaskDepend) {
6495	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
6496	OpenMPIRBuilder OMPBuilder(*M);
6497	OMPBuilder.Config.IsTargetDevice = false;
6498	OMPBuilder.initialize();
6499	F->setName("func");
6500	IRBuilder<> Builder(BB);
6501	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
6502	BasicBlock *AllocaBB = Builder.GetInsertBlock();
6503	BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "alloca.split");
6504	OpenMPIRBuilder::LocationDescription Loc(
6505	InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
6506	AllocaInst *InDep = Builder.CreateAlloca(Ty: Type::getInt32Ty(C&: M->getContext()));
6507	SmallVector<OpenMPIRBuilder::DependData> DDS;
6508	{
6509	OpenMPIRBuilder::DependData DDIn(RTLDependenceKindTy::DepIn,
6510	Type::getInt32Ty(C&: M->getContext()), InDep);
6511	DDS.push_back(Elt: DDIn);
6512	}
6513	Builder.restoreIP(IP: OMPBuilder.createTask(
6514	Loc, AllocaIP: InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB,
6515	/*Tied=*/false, /*Final*/ nullptr, /*IfCondition*/ nullptr, Dependencies: DDS));
6516	OMPBuilder.finalize();
6517	Builder.CreateRetVoid();
6518
6519	// Check for the `NumDeps` argument
6520	CallInst *TaskAllocCall = dyn_cast<CallInst>(
6521	Val: OMPBuilder
6522	.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_with_deps)
6523	->user_back());
6524	ASSERT_NE(TaskAllocCall, nullptr);
6525	ConstantInt *NumDeps = dyn_cast<ConstantInt>(Val: TaskAllocCall->getArgOperand(i: 3));
6526	ASSERT_NE(NumDeps, nullptr);
6527	EXPECT_EQ(NumDeps->getZExtValue(), 1U);
6528
6529	// Check for the `DepInfo` array argument
6530	AllocaInst *DepArray = dyn_cast<AllocaInst>(Val: TaskAllocCall->getOperand(i_nocapture: 4));
6531	ASSERT_NE(DepArray, nullptr);
6532	Value::user_iterator DepArrayI = DepArray->user_begin();
6533	++DepArrayI;
6534	Value::user_iterator DepInfoI = DepArrayI->user_begin();
6535	// Check for the `DependKind` flag in the `DepInfo` array
6536	Value *Flag = findStoredValue<GetElementPtrInst>(AllocaValue: *DepInfoI);
6537	ASSERT_NE(Flag, nullptr);
6538	ConstantInt *FlagInt = dyn_cast<ConstantInt>(Val: Flag);
6539	ASSERT_NE(FlagInt, nullptr);
6540	EXPECT_EQ(FlagInt->getZExtValue(),
6541	static_cast<unsigned int>(RTLDependenceKindTy::DepIn));
6542	++DepInfoI;
6543	// Check for the size in the `DepInfo` array
6544	Value *Size = findStoredValue<GetElementPtrInst>(AllocaValue: *DepInfoI);
6545	ASSERT_NE(Size, nullptr);
6546	ConstantInt *SizeInt = dyn_cast<ConstantInt>(Val: Size);
6547	ASSERT_NE(SizeInt, nullptr);
6548	EXPECT_EQ(SizeInt->getZExtValue(), 4U);
6549	++DepInfoI;
6550	// Check for the variable address in the `DepInfo` array
6551	Value *AddrStored = findStoredValue<GetElementPtrInst>(AllocaValue: *DepInfoI);
6552	ASSERT_NE(AddrStored, nullptr);
6553	PtrToIntInst *AddrInt = dyn_cast<PtrToIntInst>(Val: AddrStored);
6554	ASSERT_NE(AddrInt, nullptr);
6555	Value *Addr = AddrInt->getPointerOperand();
6556	EXPECT_EQ(Addr, InDep);
6557
6558	ConstantInt *NumDepsNoAlias =
6559	dyn_cast<ConstantInt>(Val: TaskAllocCall->getArgOperand(i: 5));
6560	ASSERT_NE(NumDepsNoAlias, nullptr);
6561	EXPECT_EQ(NumDepsNoAlias->getZExtValue(), 0U);
6562	EXPECT_EQ(TaskAllocCall->getOperand(6),
6563	ConstantPointerNull::get(PointerType::getUnqual(M->getContext())));
6564
6565	EXPECT_FALSE(verifyModule(*M, &errs()));
6566	}
6567
6568	TEST_F(OpenMPIRBuilderTest, CreateTaskFinal) {
6569	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
6570	OpenMPIRBuilder OMPBuilder(*M);
6571	OMPBuilder.Config.IsTargetDevice = false;
6572	OMPBuilder.initialize();
6573	F->setName("func");
6574	IRBuilder<> Builder(BB);
6575	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
6576	BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "alloca.split");
6577	IRBuilderBase::InsertPoint AllocaIP = Builder.saveIP();
6578	Builder.SetInsertPoint(BodyBB);
6579	Value *Final = Builder.CreateICmp(
6580	P: CmpInst::Predicate::ICMP_EQ, LHS: F->getArg(i: 0),
6581	RHS: ConstantInt::get(Ty: Type::getInt32Ty(C&: M->getContext()), V: 0U));
6582	OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
6583	Builder.restoreIP(IP: OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,
6584	/*Tied=*/false, Final));
6585	OMPBuilder.finalize();
6586	Builder.CreateRetVoid();
6587
6588	// Check for the `Tied` argument
6589	CallInst *TaskAllocCall = dyn_cast<CallInst>(
6590	Val: OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc)
6591	->user_back());
6592	ASSERT_NE(TaskAllocCall, nullptr);
6593	BinaryOperator *OrInst =
6594	dyn_cast<BinaryOperator>(Val: TaskAllocCall->getArgOperand(i: 2));
6595	ASSERT_NE(OrInst, nullptr);
6596	EXPECT_EQ(OrInst->getOpcode(), BinaryOperator::BinaryOps::Or);
6597
6598	// One of the arguments to `or` instruction is the tied flag, which is equal
6599	// to zero.
6600	EXPECT_TRUE(any_of(OrInst->operands(), [](Value *op) {
6601	if (ConstantInt *TiedValue = dyn_cast<ConstantInt>(op))
6602	return TiedValue->getSExtValue() == 0;
6603	return false;
6604	}));
6605
6606	// One of the arguments to `or` instruction is the final condition.
6607	EXPECT_TRUE(any_of(OrInst->operands(), [Final](Value *op) {
6608	if (SelectInst *Select = dyn_cast<SelectInst>(op)) {
6609	ConstantInt *TrueValue = dyn_cast<ConstantInt>(Select->getTrueValue());
6610	ConstantInt *FalseValue = dyn_cast<ConstantInt>(Select->getFalseValue());
6611	if (!TrueValue || !FalseValue)
6612	return false;
6613	return Select->getCondition() == Final &&
6614	TrueValue->getSExtValue() == 2 && FalseValue->getSExtValue() == 0;
6615	}
6616	return false;
6617	}));
6618
6619	EXPECT_FALSE(verifyModule(*M, &errs()));
6620	}
6621
6622	TEST_F(OpenMPIRBuilderTest, CreateTaskIfCondition) {
6623	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
6624	OpenMPIRBuilder OMPBuilder(*M);
6625	OMPBuilder.Config.IsTargetDevice = false;
6626	OMPBuilder.initialize();
6627	F->setName("func");
6628	IRBuilder<> Builder(BB);
6629	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
6630	BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "alloca.split");
6631	IRBuilderBase::InsertPoint AllocaIP = Builder.saveIP();
6632	Builder.SetInsertPoint(BodyBB);
6633	Value *IfCondition = Builder.CreateICmp(
6634	P: CmpInst::Predicate::ICMP_EQ, LHS: F->getArg(i: 0),
6635	RHS: ConstantInt::get(Ty: Type::getInt32Ty(C&: M->getContext()), V: 0U));
6636	OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
6637	Builder.restoreIP(IP: OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,
6638	/*Tied=*/false, /*Final=*/nullptr,
6639	IfCondition));
6640	OMPBuilder.finalize();
6641	Builder.CreateRetVoid();
6642
6643	EXPECT_FALSE(verifyModule(*M, &errs()));
6644
6645	CallInst *TaskAllocCall = dyn_cast<CallInst>(
6646	Val: OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc)
6647	->user_back());
6648	ASSERT_NE(TaskAllocCall, nullptr);
6649
6650	// Check the branching is based on the if condition argument.
6651	BranchInst *IfConditionBranchInst =
6652	dyn_cast<BranchInst>(Val: TaskAllocCall->getParent()->getTerminator());
6653	ASSERT_NE(IfConditionBranchInst, nullptr);
6654	ASSERT_TRUE(IfConditionBranchInst->isConditional());
6655	EXPECT_EQ(IfConditionBranchInst->getCondition(), IfCondition);
6656
6657	// Check that the `__kmpc_omp_task` executes only in the then branch.
6658	CallInst *TaskCall = dyn_cast<CallInst>(
6659	Val: OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task)
6660	->user_back());
6661	ASSERT_NE(TaskCall, nullptr);
6662	EXPECT_EQ(TaskCall->getParent(), IfConditionBranchInst->getSuccessor(0));
6663
6664	// Check that the OpenMP Runtime Functions specific to `if` clause execute
6665	// only in the else branch. Also check that the function call is between the
6666	// `__kmpc_omp_task_begin_if0` and `__kmpc_omp_task_complete_if0` calls.
6667	CallInst *TaskBeginIfCall = dyn_cast<CallInst>(
6668	Val: OMPBuilder
6669	.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_begin_if0)
6670	->user_back());
6671	CallInst *TaskCompleteCall = dyn_cast<CallInst>(
6672	Val: OMPBuilder
6673	.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_complete_if0)
6674	->user_back());
6675	ASSERT_NE(TaskBeginIfCall, nullptr);
6676	ASSERT_NE(TaskCompleteCall, nullptr);
6677	Function *OulinedFn =
6678	dyn_cast<Function>(Val: TaskAllocCall->getArgOperand(i: 5)->stripPointerCasts());
6679	ASSERT_NE(OulinedFn, nullptr);
6680	CallInst *OulinedFnCall = dyn_cast<CallInst>(Val: OulinedFn->user_back());
6681	ASSERT_NE(OulinedFnCall, nullptr);
6682	EXPECT_EQ(TaskBeginIfCall->getParent(),
6683	IfConditionBranchInst->getSuccessor(1));
6684
6685	EXPECT_EQ(TaskBeginIfCall->getNextNonDebugInstruction(), OulinedFnCall);
6686	EXPECT_EQ(OulinedFnCall->getNextNonDebugInstruction(), TaskCompleteCall);
6687	}
6688
6689	TEST_F(OpenMPIRBuilderTest, CreateTaskgroup) {
6690	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
6691	OpenMPIRBuilder OMPBuilder(*M);
6692	OMPBuilder.initialize();
6693	F->setName("func");
6694	IRBuilder<> Builder(BB);
6695
6696	AllocaInst *ValPtr32 = Builder.CreateAlloca(Ty: Builder.getInt32Ty());
6697	AllocaInst *ValPtr128 = Builder.CreateAlloca(Ty: Builder.getInt128Ty());
6698	Value *Val128 =
6699	Builder.CreateLoad(Ty: Builder.getInt128Ty(), Ptr: ValPtr128, Name: "bodygen.load");
6700	Instruction *ThenTerm, *ElseTerm;
6701
6702	Value *InternalStoreInst, *InternalLoad32, *InternalLoad128, *InternalIfCmp;
6703
6704	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
6705	Builder.restoreIP(IP: AllocaIP);
6706	AllocaInst *Local128 = Builder.CreateAlloca(Ty: Builder.getInt128Ty(), ArraySize: nullptr,
6707	Name: "bodygen.alloca128");
6708
6709	Builder.restoreIP(IP: CodeGenIP);
6710	// Loading and storing captured pointer and values
6711	InternalStoreInst = Builder.CreateStore(Val: Val128, Ptr: Local128);
6712	InternalLoad32 = Builder.CreateLoad(Ty: ValPtr32->getAllocatedType(), Ptr: ValPtr32,
6713	Name: "bodygen.load32");
6714
6715	InternalLoad128 = Builder.CreateLoad(Ty: Local128->getAllocatedType(), Ptr: Local128,
6716	Name: "bodygen.local.load128");
6717	InternalIfCmp = Builder.CreateICmpNE(
6718	LHS: InternalLoad32,
6719	RHS: Builder.CreateTrunc(V: InternalLoad128, DestTy: InternalLoad32->getType()));
6720	SplitBlockAndInsertIfThenElse(Cond: InternalIfCmp,
6721	SplitBefore: CodeGenIP.getBlock()->getTerminator(),
6722	ThenTerm: &ThenTerm, ElseTerm: &ElseTerm);
6723	};
6724
6725	BasicBlock *AllocaBB = Builder.GetInsertBlock();
6726	BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "alloca.split");
6727	OpenMPIRBuilder::LocationDescription Loc(
6728	InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
6729	Builder.restoreIP(IP: OMPBuilder.createTaskgroup(
6730	Loc, AllocaIP: InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
6731	BodyGenCB));
6732	OMPBuilder.finalize();
6733	Builder.CreateRetVoid();
6734
6735	EXPECT_FALSE(verifyModule(*M, &errs()));
6736
6737	CallInst *TaskgroupCall = dyn_cast<CallInst>(
6738	Val: OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskgroup)
6739	->user_back());
6740	ASSERT_NE(TaskgroupCall, nullptr);
6741	CallInst *EndTaskgroupCall = dyn_cast<CallInst>(
6742	Val: OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_taskgroup)
6743	->user_back());
6744	ASSERT_NE(EndTaskgroupCall, nullptr);
6745
6746	// Verify the Ident argument
6747	GlobalVariable *Ident = cast<GlobalVariable>(Val: TaskgroupCall->getArgOperand(i: 0));
6748	ASSERT_NE(Ident, nullptr);
6749	EXPECT_TRUE(Ident->hasInitializer());
6750	Constant *Initializer = Ident->getInitializer();
6751	GlobalVariable *SrcStrGlob =
6752	cast<GlobalVariable>(Val: Initializer->getOperand(i: 4)->stripPointerCasts());
6753	ASSERT_NE(SrcStrGlob, nullptr);
6754	ConstantDataArray *SrcSrc =
6755	dyn_cast<ConstantDataArray>(Val: SrcStrGlob->getInitializer());
6756	ASSERT_NE(SrcSrc, nullptr);
6757
6758	// Verify the num_threads argument.
6759	CallInst *GTID = dyn_cast<CallInst>(Val: TaskgroupCall->getArgOperand(i: 1));
6760	ASSERT_NE(GTID, nullptr);
6761	EXPECT_EQ(GTID->arg_size(), 1U);
6762	EXPECT_EQ(GTID->getCalledFunction(), OMPBuilder.getOrCreateRuntimeFunctionPtr(
6763	OMPRTL___kmpc_global_thread_num));
6764
6765	// Checking the general structure of the IR generated is same as expected.
6766	Instruction *GeneratedStoreInst = TaskgroupCall->getNextNonDebugInstruction();
6767	EXPECT_EQ(GeneratedStoreInst, InternalStoreInst);
6768	Instruction *GeneratedLoad32 =
6769	GeneratedStoreInst->getNextNonDebugInstruction();
6770	EXPECT_EQ(GeneratedLoad32, InternalLoad32);
6771	Instruction *GeneratedLoad128 = GeneratedLoad32->getNextNonDebugInstruction();
6772	EXPECT_EQ(GeneratedLoad128, InternalLoad128);
6773
6774	// Checking the ordering because of the if statements and that
6775	// `__kmp_end_taskgroup` call is after the if branching.
6776	BasicBlock *RefOrder[] = {TaskgroupCall->getParent(), ThenTerm->getParent(),
6777	ThenTerm->getSuccessor(Idx: 0),
6778	EndTaskgroupCall->getParent(),
6779	ElseTerm->getParent()};
6780	verifyDFSOrder(F, RefOrder);
6781	}
6782
6783	TEST_F(OpenMPIRBuilderTest, CreateTaskgroupWithTasks) {
6784	using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
6785	OpenMPIRBuilder OMPBuilder(*M);
6786	OMPBuilder.Config.IsTargetDevice = false;
6787	OMPBuilder.initialize();
6788	F->setName("func");
6789	IRBuilder<> Builder(BB);
6790
6791	auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
6792	Builder.restoreIP(IP: AllocaIP);
6793	AllocaInst *Alloca32 =
6794	Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr, Name: "bodygen.alloca32");
6795	AllocaInst *Alloca64 =
6796	Builder.CreateAlloca(Ty: Builder.getInt64Ty(), ArraySize: nullptr, Name: "bodygen.alloca64");
6797	Builder.restoreIP(IP: CodeGenIP);
6798	auto TaskBodyGenCB1 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
6799	Builder.restoreIP(IP: CodeGenIP);
6800	LoadInst *LoadValue =
6801	Builder.CreateLoad(Ty: Alloca64->getAllocatedType(), Ptr: Alloca64);
6802	Value *AddInst = Builder.CreateAdd(LHS: LoadValue, RHS: Builder.getInt64(C: 64));
6803	Builder.CreateStore(Val: AddInst, Ptr: Alloca64);
6804	};
6805	OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
6806	Builder.restoreIP(IP: OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB: TaskBodyGenCB1));
6807
6808	auto TaskBodyGenCB2 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
6809	Builder.restoreIP(IP: CodeGenIP);
6810	LoadInst *LoadValue =
6811	Builder.CreateLoad(Ty: Alloca32->getAllocatedType(), Ptr: Alloca32);
6812	Value *AddInst = Builder.CreateAdd(LHS: LoadValue, RHS: Builder.getInt32(C: 32));
6813	Builder.CreateStore(Val: AddInst, Ptr: Alloca32);
6814	};
6815	OpenMPIRBuilder::LocationDescription Loc2(Builder.saveIP(), DL);
6816	Builder.restoreIP(IP: OMPBuilder.createTask(Loc: Loc2, AllocaIP, BodyGenCB: TaskBodyGenCB2));
6817	};
6818
6819	BasicBlock *AllocaBB = Builder.GetInsertBlock();
6820	BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "alloca.split");
6821	OpenMPIRBuilder::LocationDescription Loc(
6822	InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
6823	Builder.restoreIP(IP: OMPBuilder.createTaskgroup(
6824	Loc, AllocaIP: InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
6825	BodyGenCB));
6826	OMPBuilder.finalize();
6827	Builder.CreateRetVoid();
6828
6829	EXPECT_FALSE(verifyModule(*M, &errs()));
6830
6831	CallInst *TaskgroupCall = dyn_cast<CallInst>(
6832	Val: OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskgroup)
6833	->user_back());
6834	ASSERT_NE(TaskgroupCall, nullptr);
6835	CallInst *EndTaskgroupCall = dyn_cast<CallInst>(
6836	Val: OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_taskgroup)
6837	->user_back());
6838	ASSERT_NE(EndTaskgroupCall, nullptr);
6839
6840	Function *TaskAllocFn =
6841	OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc);
6842	ASSERT_EQ(TaskAllocFn->getNumUses(), 2u);
6843
6844	CallInst *FirstTaskAllocCall =
6845	dyn_cast_or_null<CallInst>(Val: *TaskAllocFn->users().begin());
6846	CallInst *SecondTaskAllocCall =
6847	dyn_cast_or_null<CallInst>(Val: *TaskAllocFn->users().begin()++);
6848	ASSERT_NE(FirstTaskAllocCall, nullptr);
6849	ASSERT_NE(SecondTaskAllocCall, nullptr);
6850
6851	// Verify that the tasks have been generated in order and inside taskgroup
6852	// construct.
6853	BasicBlock *RefOrder[] = {
6854	TaskgroupCall->getParent(), FirstTaskAllocCall->getParent(),
6855	SecondTaskAllocCall->getParent(), EndTaskgroupCall->getParent()};
6856	verifyDFSOrder(F, RefOrder);
6857	}
6858
6859	TEST_F(OpenMPIRBuilderTest, EmitOffloadingArraysArguments) {
6860	OpenMPIRBuilder OMPBuilder(*M);
6861	OMPBuilder.initialize();
6862
6863	IRBuilder<> Builder(BB);
6864
6865	OpenMPIRBuilder::TargetDataRTArgs RTArgs;
6866	OpenMPIRBuilder::TargetDataInfo Info(true, false);
6867
6868	auto VoidPtrTy = PointerType::getUnqual(C&: Builder.getContext());
6869	auto VoidPtrPtrTy = PointerType::getUnqual(C&: Builder.getContext());
6870	auto Int64Ty = Type::getInt64Ty(C&: Builder.getContext());
6871	auto Int64PtrTy = PointerType::getUnqual(C&: Builder.getContext());
6872	auto Array4VoidPtrTy = ArrayType::get(ElementType: VoidPtrTy, NumElements: 4);
6873	auto Array4Int64PtrTy = ArrayType::get(ElementType: Int64Ty, NumElements: 4);
6874
6875	Info.RTArgs.BasePointersArray =
6876	ConstantPointerNull::get(T: Array4VoidPtrTy->getPointerTo(AddrSpace: 0));
6877	Info.RTArgs.PointersArray =
6878	ConstantPointerNull::get(T: Array4VoidPtrTy->getPointerTo());
6879	Info.RTArgs.SizesArray =
6880	ConstantPointerNull::get(T: Array4Int64PtrTy->getPointerTo());
6881	Info.RTArgs.MapTypesArray =
6882	ConstantPointerNull::get(T: Array4Int64PtrTy->getPointerTo());
6883	Info.RTArgs.MapNamesArray =
6884	ConstantPointerNull::get(T: Array4VoidPtrTy->getPointerTo());
6885	Info.RTArgs.MappersArray =
6886	ConstantPointerNull::get(T: Array4VoidPtrTy->getPointerTo());
6887	Info.NumberOfPtrs = 4;
6888
6889	OMPBuilder.emitOffloadingArraysArgument(Builder, RTArgs, Info, EmitDebug: false, ForEndCall: false);
6890
6891	EXPECT_NE(RTArgs.BasePointersArray, nullptr);
6892	EXPECT_NE(RTArgs.PointersArray, nullptr);
6893	EXPECT_NE(RTArgs.SizesArray, nullptr);
6894	EXPECT_NE(RTArgs.MapTypesArray, nullptr);
6895	EXPECT_NE(RTArgs.MappersArray, nullptr);
6896	EXPECT_NE(RTArgs.MapNamesArray, nullptr);
6897	EXPECT_EQ(RTArgs.MapTypesArrayEnd, nullptr);
6898
6899	EXPECT_EQ(RTArgs.BasePointersArray->getType(), VoidPtrPtrTy);
6900	EXPECT_EQ(RTArgs.PointersArray->getType(), VoidPtrPtrTy);
6901	EXPECT_EQ(RTArgs.SizesArray->getType(), Int64PtrTy);
6902	EXPECT_EQ(RTArgs.MapTypesArray->getType(), Int64PtrTy);
6903	EXPECT_EQ(RTArgs.MappersArray->getType(), VoidPtrPtrTy);
6904	EXPECT_EQ(RTArgs.MapNamesArray->getType(), VoidPtrPtrTy);
6905	}
6906
6907	TEST_F(OpenMPIRBuilderTest, OffloadEntriesInfoManager) {
6908	OpenMPIRBuilder OMPBuilder(*M);
6909	OMPBuilder.setConfig(
6910	OpenMPIRBuilderConfig(true, false, false, false, false, false, false));
6911	OffloadEntriesInfoManager &InfoManager = OMPBuilder.OffloadInfoManager;
6912	TargetRegionEntryInfo EntryInfo("parent", 1, 2, 4, 0);
6913	InfoManager.initializeTargetRegionEntryInfo(EntryInfo, Order: 0);
6914	EXPECT_TRUE(InfoManager.hasTargetRegionEntryInfo(EntryInfo));
6915	InfoManager.initializeDeviceGlobalVarEntryInfo(
6916	Name: "gvar", Flags: OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo, Order: 0);
6917	InfoManager.registerTargetRegionEntryInfo(
6918	EntryInfo, Addr: nullptr, ID: nullptr,
6919	Flags: OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion);
6920	InfoManager.registerDeviceGlobalVarEntryInfo(
6921	VarName: "gvar", Addr: 0x0, VarSize: 8, Flags: OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo,
6922	Linkage: GlobalValue::WeakAnyLinkage);
6923	EXPECT_TRUE(InfoManager.hasDeviceGlobalVarEntryInfo("gvar"));
6924	}
6925
6926	// Tests both registerTargetGlobalVariable and getAddrOfDeclareTargetVar as they
6927	// call each other (recursively in some cases). The test case test these
6928	// functions by utilising them for host code generation for declare target
6929	// global variables
6930	TEST_F(OpenMPIRBuilderTest, registerTargetGlobalVariable) {
6931	OpenMPIRBuilder OMPBuilder(*M);
6932	OMPBuilder.initialize();
6933	OpenMPIRBuilderConfig Config(false, false, false, false, false, false, false);
6934	OMPBuilder.setConfig(Config);
6935
6936	std::vector<llvm::Triple> TargetTriple;
6937	TargetTriple.emplace_back(args: "amdgcn-amd-amdhsa");
6938
6939	TargetRegionEntryInfo EntryInfo("", 42, 4711, 17);
6940	std::vector<GlobalVariable *> RefsGathered;
6941
6942	std::vector<Constant *> Globals;
6943	auto *IntTy = Type::getInt32Ty(C&: Ctx);
6944	for (int I = 0; I < 2; ++I) {
6945	Globals.push_back(x: M->getOrInsertGlobal(
6946	Name: "test_data_int_" + std::to_string(val: I), Ty: IntTy, CreateGlobalCallback: [&]() -> GlobalVariable * {
6947	return new GlobalVariable(
6948	*M, IntTy, false, GlobalValue::LinkageTypes::WeakAnyLinkage,
6949	ConstantInt::get(Ty: IntTy, V: I), "test_data_int_" + std::to_string(val: I));
6950	}));
6951	}
6952
6953	OMPBuilder.registerTargetGlobalVariable(
6954	CaptureClause: OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo,
6955	DeviceClause: OffloadEntriesInfoManager::OMPTargetDeviceClauseAny, IsDeclaration: false, IsExternallyVisible: true,
6956	EntryInfo, MangledName: Globals[0]->getName(), GeneratedRefs&: RefsGathered, OpenMPSIMD: false, TargetTriple,
6957	GlobalInitializer: nullptr, VariableLinkage: nullptr, LlvmPtrTy: Globals[0]->getType(), Addr: Globals[0]);
6958
6959	OMPBuilder.registerTargetGlobalVariable(
6960	CaptureClause: OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink,
6961	DeviceClause: OffloadEntriesInfoManager::OMPTargetDeviceClauseAny, IsDeclaration: false, IsExternallyVisible: true,
6962	EntryInfo, MangledName: Globals[1]->getName(), GeneratedRefs&: RefsGathered, OpenMPSIMD: false, TargetTriple,
6963	GlobalInitializer: nullptr, VariableLinkage: nullptr, LlvmPtrTy: Globals[1]->getType(), Addr: Globals[1]);
6964
6965	llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportfn =
6966	[](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
6967	const llvm::TargetRegionEntryInfo &EntryInfo) -> void {
6968	// If this is invoked, then we want to emit an error, even if it is not
6969	// neccesarily the most readable, as something has went wrong. The
6970	// test-suite unfortunately eats up all error output
6971	ASSERT_EQ(Kind, Kind);
6972	};
6973
6974	OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFunction&: ErrorReportfn);
6975
6976	// Clauses for data_int_0 with To + Any clauses for the host
6977	std::vector<GlobalVariable *> OffloadEntries;
6978	OffloadEntries.push_back(x: M->getNamedGlobal(Name: ".offloading.entry_name"));
6979	OffloadEntries.push_back(
6980	x: M->getNamedGlobal(Name: ".offloading.entry.test_data_int_0"));
6981
6982	// Clauses for data_int_1 with Link + Any clauses for the host
6983	OffloadEntries.push_back(
6984	x: M->getNamedGlobal(Name: "test_data_int_1_decl_tgt_ref_ptr"));
6985	OffloadEntries.push_back(x: M->getNamedGlobal(Name: ".offloading.entry_name.1"));
6986	OffloadEntries.push_back(
6987	x: M->getNamedGlobal(Name: ".offloading.entry.test_data_int_1_decl_tgt_ref_ptr"));
6988
6989	for (unsigned I = 0; I < OffloadEntries.size(); ++I)
6990	EXPECT_NE(OffloadEntries[I], nullptr);
6991
6992	// Metadata generated for the host offload module
6993	NamedMDNode *OffloadMetadata = M->getNamedMetadata(Name: "omp_offload.info");
6994	ASSERT_THAT(OffloadMetadata, testing::NotNull());
6995	StringRef Nodes[2] = {
6996	cast<MDString>(Val: OffloadMetadata->getOperand(i: 0)->getOperand(I: 1))
6997	->getString(),
6998	cast<MDString>(Val: OffloadMetadata->getOperand(i: 1)->getOperand(I: 1))
6999	->getString()};
7000	EXPECT_THAT(
7001	Nodes, testing::UnorderedElementsAre("test_data_int_0",
7002	"test_data_int_1_decl_tgt_ref_ptr"));
7003	}
7004
7005	TEST_F(OpenMPIRBuilderTest, createGPUOffloadEntry) {
7006	OpenMPIRBuilder OMPBuilder(*M);
7007	OMPBuilder.initialize();
7008	OpenMPIRBuilderConfig Config(/* IsTargetDevice = */ true,
7009	/* IsGPU = */ true,
7010	/* OpenMPOffloadMandatory = */ false,
7011	/* HasRequiresReverseOffload = */ false,
7012	/* HasRequiresUnifiedAddress = */ false,
7013	/* HasRequiresUnifiedSharedMemory = */ false,
7014	/* HasRequiresDynamicAllocators = */ false);
7015	OMPBuilder.setConfig(Config);
7016
7017	FunctionCallee FnTypeAndCallee =
7018	M->getOrInsertFunction(Name: "test_kernel", RetTy: Type::getVoidTy(C&: Ctx));
7019
7020	auto *Fn = cast<Function>(Val: FnTypeAndCallee.getCallee());
7021	OMPBuilder.createOffloadEntry(/* ID = */ nullptr, Addr: Fn,
7022	/* Size = */ 0,
7023	/* Flags = */ 0, GlobalValue::WeakAnyLinkage);
7024
7025	// Check nvvm.annotations only created for GPU kernels
7026	NamedMDNode *MD = M->getNamedMetadata(Name: "nvvm.annotations");
7027	EXPECT_NE(MD, nullptr);
7028	EXPECT_EQ(MD->getNumOperands(), 1u);
7029
7030	MDNode *Annotations = MD->getOperand(i: 0);
7031	EXPECT_EQ(Annotations->getNumOperands(), 3u);
7032
7033	Constant *ConstVal =
7034	dyn_cast<ConstantAsMetadata>(Val: Annotations->getOperand(I: 0))->getValue();
7035	EXPECT_TRUE(isa<Function>(Fn));
7036	EXPECT_EQ(ConstVal, cast<Function>(Fn));
7037
7038	EXPECT_TRUE(Annotations->getOperand(1).equalsStr("kernel"));
7039
7040	EXPECT_TRUE(mdconst::hasa<ConstantInt>(Annotations->getOperand(2)));
7041	APInt IntVal =
7042	mdconst::extract<ConstantInt>(MD: Annotations->getOperand(I: 2))->getValue();
7043	EXPECT_EQ(IntVal, 1);
7044
7045	// Check kernel attributes
7046	EXPECT_TRUE(Fn->hasFnAttribute("kernel"));
7047	EXPECT_TRUE(Fn->hasFnAttribute(Attribute::MustProgress));
7048	}
7049
7050	} // namespace
7051