1 | //===-- Transformational.cpp ------------------------------------*- C++ -*-===// |
2 | // Generate transformational intrinsic runtime API calls. |
3 | // |
4 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
5 | // See https://llvm.org/LICENSE.txt for license information. |
6 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
7 | // |
8 | //===----------------------------------------------------------------------===// |
9 | |
10 | #include "flang/Optimizer/Builder/Runtime/Transformational.h" |
11 | #include "flang/Optimizer/Builder/BoxValue.h" |
12 | #include "flang/Optimizer/Builder/Character.h" |
13 | #include "flang/Optimizer/Builder/FIRBuilder.h" |
14 | #include "flang/Optimizer/Builder/Runtime/RTBuilder.h" |
15 | #include "flang/Optimizer/Support/Utils.h" |
16 | #include "flang/Runtime/matmul-transpose.h" |
17 | #include "flang/Runtime/matmul.h" |
18 | #include "flang/Runtime/transformational.h" |
19 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
20 | |
21 | using namespace Fortran::runtime; |
22 | |
23 | /// Placeholder for real*10 version of BesselJn intrinsic. |
24 | struct ForcedBesselJn_10 { |
25 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselJn_10)); |
26 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
27 | return [](mlir::MLIRContext *ctx) { |
28 | auto ty = mlir::Float80Type::get(ctx); |
29 | auto boxTy = |
30 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
31 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
32 | auto intTy = mlir::IntegerType::get(ctx, 32); |
33 | return mlir::FunctionType::get( |
34 | ctx, {boxTy, intTy, intTy, ty, ty, ty, strTy, intTy}, {}); |
35 | }; |
36 | } |
37 | }; |
38 | |
39 | /// Placeholder for real*16 version of BesselJn intrinsic. |
40 | struct ForcedBesselJn_16 { |
41 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselJn_16)); |
42 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
43 | return [](mlir::MLIRContext *ctx) { |
44 | auto ty = mlir::Float128Type::get(ctx); |
45 | auto boxTy = |
46 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
47 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
48 | auto intTy = mlir::IntegerType::get(ctx, 32); |
49 | return mlir::FunctionType::get( |
50 | ctx, {boxTy, intTy, intTy, ty, ty, ty, strTy, intTy}, {}); |
51 | }; |
52 | } |
53 | }; |
54 | |
55 | /// Placeholder for real*10 version of BesselJn intrinsic when `x == 0.0`. |
56 | struct ForcedBesselJnX0_10 { |
57 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselJnX0_10)); |
58 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
59 | return [](mlir::MLIRContext *ctx) { |
60 | auto boxTy = |
61 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
62 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
63 | auto intTy = mlir::IntegerType::get(ctx, 32); |
64 | return mlir::FunctionType::get(ctx, {boxTy, intTy, intTy, strTy, intTy}, |
65 | {}); |
66 | }; |
67 | } |
68 | }; |
69 | |
70 | /// Placeholder for real*16 version of BesselJn intrinsic when `x == 0.0`. |
71 | struct ForcedBesselJnX0_16 { |
72 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselJnX0_16)); |
73 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
74 | return [](mlir::MLIRContext *ctx) { |
75 | auto boxTy = |
76 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
77 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
78 | auto intTy = mlir::IntegerType::get(ctx, 32); |
79 | return mlir::FunctionType::get(ctx, {boxTy, intTy, intTy, strTy, intTy}, |
80 | {}); |
81 | }; |
82 | } |
83 | }; |
84 | |
85 | /// Placeholder for real*10 version of BesselYn intrinsic. |
86 | struct ForcedBesselYn_10 { |
87 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselYn_10)); |
88 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
89 | return [](mlir::MLIRContext *ctx) { |
90 | auto ty = mlir::Float80Type::get(ctx); |
91 | auto boxTy = |
92 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
93 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
94 | auto intTy = mlir::IntegerType::get(ctx, 32); |
95 | return mlir::FunctionType::get( |
96 | ctx, {boxTy, intTy, intTy, ty, ty, ty, strTy, intTy}, {}); |
97 | }; |
98 | } |
99 | }; |
100 | |
101 | /// Placeholder for real*16 version of BesselYn intrinsic. |
102 | struct ForcedBesselYn_16 { |
103 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselYn_16)); |
104 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
105 | return [](mlir::MLIRContext *ctx) { |
106 | auto ty = mlir::Float128Type::get(ctx); |
107 | auto boxTy = |
108 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
109 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
110 | auto intTy = mlir::IntegerType::get(ctx, 32); |
111 | return mlir::FunctionType::get( |
112 | ctx, {boxTy, intTy, intTy, ty, ty, ty, strTy, intTy}, {}); |
113 | }; |
114 | } |
115 | }; |
116 | |
117 | /// Placeholder for real*10 version of BesselYn intrinsic when `x == 0.0`. |
118 | struct ForcedBesselYnX0_10 { |
119 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselYnX0_10)); |
120 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
121 | return [](mlir::MLIRContext *ctx) { |
122 | auto boxTy = |
123 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
124 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
125 | auto intTy = mlir::IntegerType::get(ctx, 32); |
126 | return mlir::FunctionType::get(ctx, {boxTy, intTy, intTy, strTy, intTy}, |
127 | {}); |
128 | }; |
129 | } |
130 | }; |
131 | |
132 | /// Placeholder for real*16 version of BesselYn intrinsic when `x == 0.0`. |
133 | struct ForcedBesselYnX0_16 { |
134 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselYnX0_16)); |
135 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
136 | return [](mlir::MLIRContext *ctx) { |
137 | auto boxTy = |
138 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
139 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
140 | auto intTy = mlir::IntegerType::get(ctx, 32); |
141 | return mlir::FunctionType::get(ctx, {boxTy, intTy, intTy, strTy, intTy}, |
142 | {}); |
143 | }; |
144 | } |
145 | }; |
146 | |
147 | /// Generate call to `BesselJn` intrinsic. |
148 | void fir::runtime::genBesselJn(fir::FirOpBuilder &builder, mlir::Location loc, |
149 | mlir::Value resultBox, mlir::Value n1, |
150 | mlir::Value n2, mlir::Value x, mlir::Value bn2, |
151 | mlir::Value bn2_1) { |
152 | mlir::func::FuncOp func; |
153 | auto xTy = x.getType(); |
154 | |
155 | if (xTy.isF32()) |
156 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselJn_4)>(loc, builder); |
157 | else if (xTy.isF64()) |
158 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselJn_8)>(loc, builder); |
159 | else if (xTy.isF80()) |
160 | func = fir::runtime::getRuntimeFunc<ForcedBesselJn_10>(loc, builder); |
161 | else if (xTy.isF128()) |
162 | func = fir::runtime::getRuntimeFunc<ForcedBesselJn_16>(loc, builder); |
163 | else |
164 | fir::intrinsicTypeTODO(builder, xTy, loc, "BESSEL_JN" ); |
165 | |
166 | auto fTy = func.getFunctionType(); |
167 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
168 | auto sourceLine = |
169 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(7)); |
170 | auto args = |
171 | fir::runtime::createArguments(builder, loc, fTy, resultBox, n1, n2, x, |
172 | bn2, bn2_1, sourceFile, sourceLine); |
173 | builder.create<fir::CallOp>(loc, func, args); |
174 | } |
175 | |
176 | /// Generate call to `BesselJn` intrinsic. This is used when `x == 0.0`. |
177 | void fir::runtime::genBesselJnX0(fir::FirOpBuilder &builder, mlir::Location loc, |
178 | mlir::Type xTy, mlir::Value resultBox, |
179 | mlir::Value n1, mlir::Value n2) { |
180 | mlir::func::FuncOp func; |
181 | |
182 | if (xTy.isF32()) |
183 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselJnX0_4)>(loc, builder); |
184 | else if (xTy.isF64()) |
185 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselJnX0_8)>(loc, builder); |
186 | else if (xTy.isF80()) |
187 | func = fir::runtime::getRuntimeFunc<ForcedBesselJnX0_10>(loc, builder); |
188 | else if (xTy.isF128()) |
189 | func = fir::runtime::getRuntimeFunc<ForcedBesselJnX0_16>(loc, builder); |
190 | else |
191 | fir::intrinsicTypeTODO(builder, xTy, loc, "BESSEL_JN" ); |
192 | |
193 | auto fTy = func.getFunctionType(); |
194 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
195 | auto sourceLine = |
196 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(4)); |
197 | auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox, n1, |
198 | n2, sourceFile, sourceLine); |
199 | builder.create<fir::CallOp>(loc, func, args); |
200 | } |
201 | |
202 | /// Generate call to `BesselYn` intrinsic. |
203 | void fir::runtime::genBesselYn(fir::FirOpBuilder &builder, mlir::Location loc, |
204 | mlir::Value resultBox, mlir::Value n1, |
205 | mlir::Value n2, mlir::Value x, mlir::Value bn1, |
206 | mlir::Value bn1_1) { |
207 | mlir::func::FuncOp func; |
208 | auto xTy = x.getType(); |
209 | |
210 | if (xTy.isF32()) |
211 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselYn_4)>(loc, builder); |
212 | else if (xTy.isF64()) |
213 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselYn_8)>(loc, builder); |
214 | else if (xTy.isF80()) |
215 | func = fir::runtime::getRuntimeFunc<ForcedBesselYn_10>(loc, builder); |
216 | else if (xTy.isF128()) |
217 | func = fir::runtime::getRuntimeFunc<ForcedBesselYn_16>(loc, builder); |
218 | else |
219 | fir::intrinsicTypeTODO(builder, xTy, loc, "BESSEL_YN" ); |
220 | |
221 | auto fTy = func.getFunctionType(); |
222 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
223 | auto sourceLine = |
224 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(7)); |
225 | auto args = |
226 | fir::runtime::createArguments(builder, loc, fTy, resultBox, n1, n2, x, |
227 | bn1, bn1_1, sourceFile, sourceLine); |
228 | builder.create<fir::CallOp>(loc, func, args); |
229 | } |
230 | |
231 | /// Generate call to `BesselYn` intrinsic. This is used when `x == 0.0`. |
232 | void fir::runtime::genBesselYnX0(fir::FirOpBuilder &builder, mlir::Location loc, |
233 | mlir::Type xTy, mlir::Value resultBox, |
234 | mlir::Value n1, mlir::Value n2) { |
235 | mlir::func::FuncOp func; |
236 | |
237 | if (xTy.isF32()) |
238 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselYnX0_4)>(loc, builder); |
239 | else if (xTy.isF64()) |
240 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselYnX0_8)>(loc, builder); |
241 | else if (xTy.isF80()) |
242 | func = fir::runtime::getRuntimeFunc<ForcedBesselYnX0_10>(loc, builder); |
243 | else if (xTy.isF128()) |
244 | func = fir::runtime::getRuntimeFunc<ForcedBesselYnX0_16>(loc, builder); |
245 | else |
246 | fir::intrinsicTypeTODO(builder, xTy, loc, "BESSEL_YN" ); |
247 | |
248 | auto fTy = func.getFunctionType(); |
249 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
250 | auto sourceLine = |
251 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(4)); |
252 | auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox, n1, |
253 | n2, sourceFile, sourceLine); |
254 | builder.create<fir::CallOp>(loc, func, args); |
255 | } |
256 | |
257 | /// Generate call to Cshift intrinsic |
258 | void fir::runtime::genCshift(fir::FirOpBuilder &builder, mlir::Location loc, |
259 | mlir::Value resultBox, mlir::Value arrayBox, |
260 | mlir::Value shiftBox, mlir::Value dimBox) { |
261 | auto cshiftFunc = fir::runtime::getRuntimeFunc<mkRTKey(Cshift)>(loc, builder); |
262 | auto fTy = cshiftFunc.getFunctionType(); |
263 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
264 | auto sourceLine = |
265 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(5)); |
266 | auto args = |
267 | fir::runtime::createArguments(builder, loc, fTy, resultBox, arrayBox, |
268 | shiftBox, dimBox, sourceFile, sourceLine); |
269 | builder.create<fir::CallOp>(loc, cshiftFunc, args); |
270 | } |
271 | |
272 | /// Generate call to the vector version of the Cshift intrinsic |
273 | void fir::runtime::genCshiftVector(fir::FirOpBuilder &builder, |
274 | mlir::Location loc, mlir::Value resultBox, |
275 | mlir::Value arrayBox, mlir::Value shiftBox) { |
276 | auto cshiftFunc = |
277 | fir::runtime::getRuntimeFunc<mkRTKey(CshiftVector)>(loc, builder); |
278 | auto fTy = cshiftFunc.getFunctionType(); |
279 | |
280 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
281 | auto sourceLine = |
282 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(4)); |
283 | auto args = fir::runtime::createArguments( |
284 | builder, loc, fTy, resultBox, arrayBox, shiftBox, sourceFile, sourceLine); |
285 | builder.create<fir::CallOp>(loc, cshiftFunc, args); |
286 | } |
287 | |
288 | /// Generate call to Eoshift intrinsic |
289 | void fir::runtime::genEoshift(fir::FirOpBuilder &builder, mlir::Location loc, |
290 | mlir::Value resultBox, mlir::Value arrayBox, |
291 | mlir::Value shiftBox, mlir::Value boundBox, |
292 | mlir::Value dimBox) { |
293 | auto eoshiftFunc = |
294 | fir::runtime::getRuntimeFunc<mkRTKey(Eoshift)>(loc, builder); |
295 | auto fTy = eoshiftFunc.getFunctionType(); |
296 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
297 | auto sourceLine = |
298 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(6)); |
299 | auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox, |
300 | arrayBox, shiftBox, boundBox, |
301 | dimBox, sourceFile, sourceLine); |
302 | builder.create<fir::CallOp>(loc, eoshiftFunc, args); |
303 | } |
304 | |
305 | /// Generate call to the vector version of the Eoshift intrinsic |
306 | void fir::runtime::genEoshiftVector(fir::FirOpBuilder &builder, |
307 | mlir::Location loc, mlir::Value resultBox, |
308 | mlir::Value arrayBox, mlir::Value shiftBox, |
309 | mlir::Value boundBox) { |
310 | auto eoshiftFunc = |
311 | fir::runtime::getRuntimeFunc<mkRTKey(EoshiftVector)>(loc, builder); |
312 | auto fTy = eoshiftFunc.getFunctionType(); |
313 | |
314 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
315 | auto sourceLine = |
316 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(5)); |
317 | |
318 | auto args = |
319 | fir::runtime::createArguments(builder, loc, fTy, resultBox, arrayBox, |
320 | shiftBox, boundBox, sourceFile, sourceLine); |
321 | builder.create<fir::CallOp>(loc, eoshiftFunc, args); |
322 | } |
323 | |
324 | /// Define ForcedMatmul<ACAT><AKIND><BCAT><BKIND> models. |
325 | struct ForcedMatmulTypeModel { |
326 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
327 | return [](mlir::MLIRContext *ctx) { |
328 | auto boxRefTy = |
329 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
330 | auto boxTy = |
331 | fir::runtime::getModel<const Fortran::runtime::Descriptor &>()(ctx); |
332 | auto strTy = fir::runtime::getModel<const char *>()(ctx); |
333 | auto intTy = fir::runtime::getModel<int>()(ctx); |
334 | return mlir::FunctionType::get( |
335 | ctx, {boxRefTy, boxTy, boxTy, strTy, intTy}, {}); |
336 | }; |
337 | } |
338 | }; |
339 | |
340 | #define MATMUL_INSTANCE(ACAT, AKIND, BCAT, BKIND) \ |
341 | struct ForcedMatmul##ACAT##AKIND##BCAT##BKIND \ |
342 | : public ForcedMatmulTypeModel { \ |
343 | static constexpr const char *name = \ |
344 | ExpandAndQuoteKey(RTNAME(Matmul##ACAT##AKIND##BCAT##BKIND)); \ |
345 | }; |
346 | |
347 | #define MATMUL_DIRECT_INSTANCE(ACAT, AKIND, BCAT, BKIND) |
348 | #define MATMUL_FORCE_ALL_TYPES 1 |
349 | |
350 | #include "flang/Runtime/matmul-instances.inc" |
351 | |
352 | /// Generate call to Matmul intrinsic runtime routine. |
353 | void fir::runtime::genMatmul(fir::FirOpBuilder &builder, mlir::Location loc, |
354 | mlir::Value resultBox, mlir::Value matrixABox, |
355 | mlir::Value matrixBBox) { |
356 | mlir::func::FuncOp func; |
357 | auto boxATy = matrixABox.getType(); |
358 | auto arrATy = fir::dyn_cast_ptrOrBoxEleTy(boxATy); |
359 | auto arrAEleTy = mlir::cast<fir::SequenceType>(arrATy).getElementType(); |
360 | auto [aCat, aKind] = fir::mlirTypeToCategoryKind(loc, arrAEleTy); |
361 | auto boxBTy = matrixBBox.getType(); |
362 | auto arrBTy = fir::dyn_cast_ptrOrBoxEleTy(boxBTy); |
363 | auto arrBEleTy = mlir::cast<fir::SequenceType>(arrBTy).getElementType(); |
364 | auto [bCat, bKind] = fir::mlirTypeToCategoryKind(loc, arrBEleTy); |
365 | |
366 | // Unsigned is treated as Integer when both operands are unsigned/integer |
367 | #define MATMUL_INSTANCE(ACAT, AKIND, BCAT, BKIND) \ |
368 | if (!func && aKind == AKIND && bKind == BKIND && \ |
369 | ((aCat == TypeCategory::ACAT && bCat == TypeCategory::BCAT) || \ |
370 | ((aCat == TypeCategory::Integer || aCat == TypeCategory::Unsigned) && \ |
371 | (bCat == TypeCategory::Integer || bCat == TypeCategory::Unsigned)))) { \ |
372 | func = \ |
373 | fir::runtime::getRuntimeFunc<ForcedMatmul##ACAT##AKIND##BCAT##BKIND>( \ |
374 | loc, builder); \ |
375 | } |
376 | |
377 | #define MATMUL_DIRECT_INSTANCE(ACAT, AKIND, BCAT, BKIND) |
378 | #define MATMUL_FORCE_ALL_TYPES 1 |
379 | #include "flang/Runtime/matmul-instances.inc" |
380 | |
381 | if (!func) { |
382 | fir::intrinsicTypeTODO2(builder, arrAEleTy, arrBEleTy, loc, "MATMUL" ); |
383 | } |
384 | auto fTy = func.getFunctionType(); |
385 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
386 | auto sourceLine = |
387 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(4)); |
388 | auto args = |
389 | fir::runtime::createArguments(builder, loc, fTy, resultBox, matrixABox, |
390 | matrixBBox, sourceFile, sourceLine); |
391 | builder.create<fir::CallOp>(loc, func, args); |
392 | } |
393 | |
394 | /// Define ForcedMatmulTranspose<ACAT><AKIND><BCAT><BKIND> models. |
395 | #define MATMUL_INSTANCE(ACAT, AKIND, BCAT, BKIND) \ |
396 | struct ForcedMatmulTranspose##ACAT##AKIND##BCAT##BKIND \ |
397 | : public ForcedMatmulTypeModel { \ |
398 | static constexpr const char *name = \ |
399 | ExpandAndQuoteKey(RTNAME(MatmulTranspose##ACAT##AKIND##BCAT##BKIND)); \ |
400 | }; |
401 | |
402 | #define MATMUL_DIRECT_INSTANCE(ACAT, AKIND, BCAT, BKIND) |
403 | #define MATMUL_FORCE_ALL_TYPES 1 |
404 | |
405 | #include "flang/Runtime/matmul-instances.inc" |
406 | |
407 | void fir::runtime::genMatmulTranspose(fir::FirOpBuilder &builder, |
408 | mlir::Location loc, mlir::Value resultBox, |
409 | mlir::Value matrixABox, |
410 | mlir::Value matrixBBox) { |
411 | mlir::func::FuncOp func; |
412 | auto boxATy = matrixABox.getType(); |
413 | auto arrATy = fir::dyn_cast_ptrOrBoxEleTy(boxATy); |
414 | auto arrAEleTy = mlir::cast<fir::SequenceType>(arrATy).getElementType(); |
415 | auto [aCat, aKind] = fir::mlirTypeToCategoryKind(loc, arrAEleTy); |
416 | auto boxBTy = matrixBBox.getType(); |
417 | auto arrBTy = fir::dyn_cast_ptrOrBoxEleTy(boxBTy); |
418 | auto arrBEleTy = mlir::cast<fir::SequenceType>(arrBTy).getElementType(); |
419 | auto [bCat, bKind] = fir::mlirTypeToCategoryKind(loc, arrBEleTy); |
420 | |
421 | #define MATMUL_INSTANCE(ACAT, AKIND, BCAT, BKIND) \ |
422 | if (!func && aCat == TypeCategory::ACAT && aKind == AKIND && \ |
423 | bCat == TypeCategory::BCAT && bKind == BKIND) { \ |
424 | func = fir::runtime::getRuntimeFunc< \ |
425 | ForcedMatmulTranspose##ACAT##AKIND##BCAT##BKIND>(loc, builder); \ |
426 | } |
427 | |
428 | #define MATMUL_DIRECT_INSTANCE(ACAT, AKIND, BCAT, BKIND) |
429 | #define MATMUL_FORCE_ALL_TYPES 1 |
430 | #include "flang/Runtime/matmul-instances.inc" |
431 | |
432 | if (!func) { |
433 | fir::intrinsicTypeTODO2(builder, arrAEleTy, arrBEleTy, loc, |
434 | "MATMUL-TRANSPOSE" ); |
435 | } |
436 | auto fTy = func.getFunctionType(); |
437 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
438 | auto sourceLine = |
439 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(4)); |
440 | auto args = |
441 | fir::runtime::createArguments(builder, loc, fTy, resultBox, matrixABox, |
442 | matrixBBox, sourceFile, sourceLine); |
443 | builder.create<fir::CallOp>(loc, func, args); |
444 | } |
445 | |
446 | /// Generate call to Pack intrinsic runtime routine. |
447 | void fir::runtime::genPack(fir::FirOpBuilder &builder, mlir::Location loc, |
448 | mlir::Value resultBox, mlir::Value arrayBox, |
449 | mlir::Value maskBox, mlir::Value vectorBox) { |
450 | auto packFunc = fir::runtime::getRuntimeFunc<mkRTKey(Pack)>(loc, builder); |
451 | auto fTy = packFunc.getFunctionType(); |
452 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
453 | auto sourceLine = |
454 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(5)); |
455 | auto args = |
456 | fir::runtime::createArguments(builder, loc, fTy, resultBox, arrayBox, |
457 | maskBox, vectorBox, sourceFile, sourceLine); |
458 | builder.create<fir::CallOp>(loc, packFunc, args); |
459 | } |
460 | |
461 | /// Generate call to Reshape intrinsic runtime routine. |
462 | void fir::runtime::genReshape(fir::FirOpBuilder &builder, mlir::Location loc, |
463 | mlir::Value resultBox, mlir::Value sourceBox, |
464 | mlir::Value shapeBox, mlir::Value padBox, |
465 | mlir::Value orderBox) { |
466 | auto func = fir::runtime::getRuntimeFunc<mkRTKey(Reshape)>(loc, builder); |
467 | auto fTy = func.getFunctionType(); |
468 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
469 | auto sourceLine = |
470 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(6)); |
471 | auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox, |
472 | sourceBox, shapeBox, padBox, |
473 | orderBox, sourceFile, sourceLine); |
474 | builder.create<fir::CallOp>(loc, func, args); |
475 | } |
476 | |
477 | /// Generate call to ShallowCopy[Direct] runtime routine. |
478 | /// ShallowCopyDirect is used iff \p resultIsAllocated is true. |
479 | void fir::runtime::genShallowCopy(fir::FirOpBuilder &builder, |
480 | mlir::Location loc, mlir::Value resultBox, |
481 | mlir::Value arrayBox, |
482 | bool resultIsAllocated) { |
483 | auto packFunc = |
484 | resultIsAllocated |
485 | ? fir::runtime::getRuntimeFunc<mkRTKey(ShallowCopyDirect)>(loc, |
486 | builder) |
487 | : fir::runtime::getRuntimeFunc<mkRTKey(ShallowCopy)>(loc, builder); |
488 | auto fTy = packFunc.getFunctionType(); |
489 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
490 | auto sourceLine = |
491 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(3)); |
492 | auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox, |
493 | arrayBox, sourceFile, sourceLine); |
494 | builder.create<fir::CallOp>(loc, packFunc, args); |
495 | } |
496 | |
497 | /// Generate call to Spread intrinsic runtime routine. |
498 | void fir::runtime::genSpread(fir::FirOpBuilder &builder, mlir::Location loc, |
499 | mlir::Value resultBox, mlir::Value sourceBox, |
500 | mlir::Value dim, mlir::Value ncopies) { |
501 | auto func = fir::runtime::getRuntimeFunc<mkRTKey(Spread)>(loc, builder); |
502 | auto fTy = func.getFunctionType(); |
503 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
504 | auto sourceLine = |
505 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(5)); |
506 | auto args = |
507 | fir::runtime::createArguments(builder, loc, fTy, resultBox, sourceBox, |
508 | dim, ncopies, sourceFile, sourceLine); |
509 | builder.create<fir::CallOp>(loc, func, args); |
510 | } |
511 | |
512 | /// Generate call to Transpose intrinsic runtime routine. |
513 | void fir::runtime::genTranspose(fir::FirOpBuilder &builder, mlir::Location loc, |
514 | mlir::Value resultBox, mlir::Value sourceBox) { |
515 | auto func = fir::runtime::getRuntimeFunc<mkRTKey(Transpose)>(loc, builder); |
516 | auto fTy = func.getFunctionType(); |
517 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
518 | auto sourceLine = |
519 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(3)); |
520 | auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox, |
521 | sourceBox, sourceFile, sourceLine); |
522 | builder.create<fir::CallOp>(loc, func, args); |
523 | } |
524 | |
525 | /// Generate call to Unpack intrinsic runtime routine. |
526 | void fir::runtime::genUnpack(fir::FirOpBuilder &builder, mlir::Location loc, |
527 | mlir::Value resultBox, mlir::Value vectorBox, |
528 | mlir::Value maskBox, mlir::Value fieldBox) { |
529 | auto unpackFunc = fir::runtime::getRuntimeFunc<mkRTKey(Unpack)>(loc, builder); |
530 | auto fTy = unpackFunc.getFunctionType(); |
531 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
532 | auto sourceLine = |
533 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(5)); |
534 | auto args = |
535 | fir::runtime::createArguments(builder, loc, fTy, resultBox, vectorBox, |
536 | maskBox, fieldBox, sourceFile, sourceLine); |
537 | builder.create<fir::CallOp>(loc, unpackFunc, args); |
538 | } |
539 | |