| 1 | //===-- Transformational.cpp ------------------------------------*- C++ -*-===// |
| 2 | // Generate transformational intrinsic runtime API calls. |
| 3 | // |
| 4 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 5 | // See https://llvm.org/LICENSE.txt for license information. |
| 6 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | |
| 10 | #include "flang/Optimizer/Builder/Runtime/Transformational.h" |
| 11 | #include "flang/Optimizer/Builder/BoxValue.h" |
| 12 | #include "flang/Optimizer/Builder/Character.h" |
| 13 | #include "flang/Optimizer/Builder/FIRBuilder.h" |
| 14 | #include "flang/Optimizer/Builder/Runtime/RTBuilder.h" |
| 15 | #include "flang/Optimizer/Support/Utils.h" |
| 16 | #include "flang/Runtime/matmul-transpose.h" |
| 17 | #include "flang/Runtime/matmul.h" |
| 18 | #include "flang/Runtime/transformational.h" |
| 19 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 20 | |
| 21 | using namespace Fortran::runtime; |
| 22 | |
| 23 | /// Placeholder for real*10 version of BesselJn intrinsic. |
| 24 | struct ForcedBesselJn_10 { |
| 25 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselJn_10)); |
| 26 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
| 27 | return [](mlir::MLIRContext *ctx) { |
| 28 | auto ty = mlir::Float80Type::get(ctx); |
| 29 | auto boxTy = |
| 30 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
| 31 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
| 32 | auto intTy = mlir::IntegerType::get(ctx, 32); |
| 33 | return mlir::FunctionType::get( |
| 34 | ctx, {boxTy, intTy, intTy, ty, ty, ty, strTy, intTy}, {}); |
| 35 | }; |
| 36 | } |
| 37 | }; |
| 38 | |
| 39 | /// Placeholder for real*16 version of BesselJn intrinsic. |
| 40 | struct ForcedBesselJn_16 { |
| 41 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselJn_16)); |
| 42 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
| 43 | return [](mlir::MLIRContext *ctx) { |
| 44 | auto ty = mlir::Float128Type::get(ctx); |
| 45 | auto boxTy = |
| 46 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
| 47 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
| 48 | auto intTy = mlir::IntegerType::get(ctx, 32); |
| 49 | return mlir::FunctionType::get( |
| 50 | ctx, {boxTy, intTy, intTy, ty, ty, ty, strTy, intTy}, {}); |
| 51 | }; |
| 52 | } |
| 53 | }; |
| 54 | |
| 55 | /// Placeholder for real*10 version of BesselJn intrinsic when `x == 0.0`. |
| 56 | struct ForcedBesselJnX0_10 { |
| 57 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselJnX0_10)); |
| 58 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
| 59 | return [](mlir::MLIRContext *ctx) { |
| 60 | auto boxTy = |
| 61 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
| 62 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
| 63 | auto intTy = mlir::IntegerType::get(ctx, 32); |
| 64 | return mlir::FunctionType::get(ctx, {boxTy, intTy, intTy, strTy, intTy}, |
| 65 | {}); |
| 66 | }; |
| 67 | } |
| 68 | }; |
| 69 | |
| 70 | /// Placeholder for real*16 version of BesselJn intrinsic when `x == 0.0`. |
| 71 | struct ForcedBesselJnX0_16 { |
| 72 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselJnX0_16)); |
| 73 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
| 74 | return [](mlir::MLIRContext *ctx) { |
| 75 | auto boxTy = |
| 76 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
| 77 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
| 78 | auto intTy = mlir::IntegerType::get(ctx, 32); |
| 79 | return mlir::FunctionType::get(ctx, {boxTy, intTy, intTy, strTy, intTy}, |
| 80 | {}); |
| 81 | }; |
| 82 | } |
| 83 | }; |
| 84 | |
| 85 | /// Placeholder for real*10 version of BesselYn intrinsic. |
| 86 | struct ForcedBesselYn_10 { |
| 87 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselYn_10)); |
| 88 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
| 89 | return [](mlir::MLIRContext *ctx) { |
| 90 | auto ty = mlir::Float80Type::get(ctx); |
| 91 | auto boxTy = |
| 92 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
| 93 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
| 94 | auto intTy = mlir::IntegerType::get(ctx, 32); |
| 95 | return mlir::FunctionType::get( |
| 96 | ctx, {boxTy, intTy, intTy, ty, ty, ty, strTy, intTy}, {}); |
| 97 | }; |
| 98 | } |
| 99 | }; |
| 100 | |
| 101 | /// Placeholder for real*16 version of BesselYn intrinsic. |
| 102 | struct ForcedBesselYn_16 { |
| 103 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselYn_16)); |
| 104 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
| 105 | return [](mlir::MLIRContext *ctx) { |
| 106 | auto ty = mlir::Float128Type::get(ctx); |
| 107 | auto boxTy = |
| 108 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
| 109 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
| 110 | auto intTy = mlir::IntegerType::get(ctx, 32); |
| 111 | return mlir::FunctionType::get( |
| 112 | ctx, {boxTy, intTy, intTy, ty, ty, ty, strTy, intTy}, {}); |
| 113 | }; |
| 114 | } |
| 115 | }; |
| 116 | |
| 117 | /// Placeholder for real*10 version of BesselYn intrinsic when `x == 0.0`. |
| 118 | struct ForcedBesselYnX0_10 { |
| 119 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselYnX0_10)); |
| 120 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
| 121 | return [](mlir::MLIRContext *ctx) { |
| 122 | auto boxTy = |
| 123 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
| 124 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
| 125 | auto intTy = mlir::IntegerType::get(ctx, 32); |
| 126 | return mlir::FunctionType::get(ctx, {boxTy, intTy, intTy, strTy, intTy}, |
| 127 | {}); |
| 128 | }; |
| 129 | } |
| 130 | }; |
| 131 | |
| 132 | /// Placeholder for real*16 version of BesselYn intrinsic when `x == 0.0`. |
| 133 | struct ForcedBesselYnX0_16 { |
| 134 | static constexpr const char *name = ExpandAndQuoteKey(RTNAME(BesselYnX0_16)); |
| 135 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
| 136 | return [](mlir::MLIRContext *ctx) { |
| 137 | auto boxTy = |
| 138 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
| 139 | auto strTy = fir::ReferenceType::get(mlir::IntegerType::get(ctx, 8)); |
| 140 | auto intTy = mlir::IntegerType::get(ctx, 32); |
| 141 | return mlir::FunctionType::get(ctx, {boxTy, intTy, intTy, strTy, intTy}, |
| 142 | {}); |
| 143 | }; |
| 144 | } |
| 145 | }; |
| 146 | |
| 147 | /// Generate call to `BesselJn` intrinsic. |
| 148 | void fir::runtime::genBesselJn(fir::FirOpBuilder &builder, mlir::Location loc, |
| 149 | mlir::Value resultBox, mlir::Value n1, |
| 150 | mlir::Value n2, mlir::Value x, mlir::Value bn2, |
| 151 | mlir::Value bn2_1) { |
| 152 | mlir::func::FuncOp func; |
| 153 | auto xTy = x.getType(); |
| 154 | |
| 155 | if (xTy.isF32()) |
| 156 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselJn_4)>(loc, builder); |
| 157 | else if (xTy.isF64()) |
| 158 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselJn_8)>(loc, builder); |
| 159 | else if (xTy.isF80()) |
| 160 | func = fir::runtime::getRuntimeFunc<ForcedBesselJn_10>(loc, builder); |
| 161 | else if (xTy.isF128()) |
| 162 | func = fir::runtime::getRuntimeFunc<ForcedBesselJn_16>(loc, builder); |
| 163 | else |
| 164 | fir::intrinsicTypeTODO(builder, xTy, loc, "BESSEL_JN" ); |
| 165 | |
| 166 | auto fTy = func.getFunctionType(); |
| 167 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 168 | auto sourceLine = |
| 169 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(7)); |
| 170 | auto args = |
| 171 | fir::runtime::createArguments(builder, loc, fTy, resultBox, n1, n2, x, |
| 172 | bn2, bn2_1, sourceFile, sourceLine); |
| 173 | builder.create<fir::CallOp>(loc, func, args); |
| 174 | } |
| 175 | |
| 176 | /// Generate call to `BesselJn` intrinsic. This is used when `x == 0.0`. |
| 177 | void fir::runtime::genBesselJnX0(fir::FirOpBuilder &builder, mlir::Location loc, |
| 178 | mlir::Type xTy, mlir::Value resultBox, |
| 179 | mlir::Value n1, mlir::Value n2) { |
| 180 | mlir::func::FuncOp func; |
| 181 | |
| 182 | if (xTy.isF32()) |
| 183 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselJnX0_4)>(loc, builder); |
| 184 | else if (xTy.isF64()) |
| 185 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselJnX0_8)>(loc, builder); |
| 186 | else if (xTy.isF80()) |
| 187 | func = fir::runtime::getRuntimeFunc<ForcedBesselJnX0_10>(loc, builder); |
| 188 | else if (xTy.isF128()) |
| 189 | func = fir::runtime::getRuntimeFunc<ForcedBesselJnX0_16>(loc, builder); |
| 190 | else |
| 191 | fir::intrinsicTypeTODO(builder, xTy, loc, "BESSEL_JN" ); |
| 192 | |
| 193 | auto fTy = func.getFunctionType(); |
| 194 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 195 | auto sourceLine = |
| 196 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(4)); |
| 197 | auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox, n1, |
| 198 | n2, sourceFile, sourceLine); |
| 199 | builder.create<fir::CallOp>(loc, func, args); |
| 200 | } |
| 201 | |
| 202 | /// Generate call to `BesselYn` intrinsic. |
| 203 | void fir::runtime::genBesselYn(fir::FirOpBuilder &builder, mlir::Location loc, |
| 204 | mlir::Value resultBox, mlir::Value n1, |
| 205 | mlir::Value n2, mlir::Value x, mlir::Value bn1, |
| 206 | mlir::Value bn1_1) { |
| 207 | mlir::func::FuncOp func; |
| 208 | auto xTy = x.getType(); |
| 209 | |
| 210 | if (xTy.isF32()) |
| 211 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselYn_4)>(loc, builder); |
| 212 | else if (xTy.isF64()) |
| 213 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselYn_8)>(loc, builder); |
| 214 | else if (xTy.isF80()) |
| 215 | func = fir::runtime::getRuntimeFunc<ForcedBesselYn_10>(loc, builder); |
| 216 | else if (xTy.isF128()) |
| 217 | func = fir::runtime::getRuntimeFunc<ForcedBesselYn_16>(loc, builder); |
| 218 | else |
| 219 | fir::intrinsicTypeTODO(builder, xTy, loc, "BESSEL_YN" ); |
| 220 | |
| 221 | auto fTy = func.getFunctionType(); |
| 222 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 223 | auto sourceLine = |
| 224 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(7)); |
| 225 | auto args = |
| 226 | fir::runtime::createArguments(builder, loc, fTy, resultBox, n1, n2, x, |
| 227 | bn1, bn1_1, sourceFile, sourceLine); |
| 228 | builder.create<fir::CallOp>(loc, func, args); |
| 229 | } |
| 230 | |
| 231 | /// Generate call to `BesselYn` intrinsic. This is used when `x == 0.0`. |
| 232 | void fir::runtime::genBesselYnX0(fir::FirOpBuilder &builder, mlir::Location loc, |
| 233 | mlir::Type xTy, mlir::Value resultBox, |
| 234 | mlir::Value n1, mlir::Value n2) { |
| 235 | mlir::func::FuncOp func; |
| 236 | |
| 237 | if (xTy.isF32()) |
| 238 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselYnX0_4)>(loc, builder); |
| 239 | else if (xTy.isF64()) |
| 240 | func = fir::runtime::getRuntimeFunc<mkRTKey(BesselYnX0_8)>(loc, builder); |
| 241 | else if (xTy.isF80()) |
| 242 | func = fir::runtime::getRuntimeFunc<ForcedBesselYnX0_10>(loc, builder); |
| 243 | else if (xTy.isF128()) |
| 244 | func = fir::runtime::getRuntimeFunc<ForcedBesselYnX0_16>(loc, builder); |
| 245 | else |
| 246 | fir::intrinsicTypeTODO(builder, xTy, loc, "BESSEL_YN" ); |
| 247 | |
| 248 | auto fTy = func.getFunctionType(); |
| 249 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 250 | auto sourceLine = |
| 251 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(4)); |
| 252 | auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox, n1, |
| 253 | n2, sourceFile, sourceLine); |
| 254 | builder.create<fir::CallOp>(loc, func, args); |
| 255 | } |
| 256 | |
| 257 | /// Generate call to Cshift intrinsic |
| 258 | void fir::runtime::genCshift(fir::FirOpBuilder &builder, mlir::Location loc, |
| 259 | mlir::Value resultBox, mlir::Value arrayBox, |
| 260 | mlir::Value shiftBox, mlir::Value dimBox) { |
| 261 | auto cshiftFunc = fir::runtime::getRuntimeFunc<mkRTKey(Cshift)>(loc, builder); |
| 262 | auto fTy = cshiftFunc.getFunctionType(); |
| 263 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 264 | auto sourceLine = |
| 265 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(5)); |
| 266 | auto args = |
| 267 | fir::runtime::createArguments(builder, loc, fTy, resultBox, arrayBox, |
| 268 | shiftBox, dimBox, sourceFile, sourceLine); |
| 269 | builder.create<fir::CallOp>(loc, cshiftFunc, args); |
| 270 | } |
| 271 | |
| 272 | /// Generate call to the vector version of the Cshift intrinsic |
| 273 | void fir::runtime::genCshiftVector(fir::FirOpBuilder &builder, |
| 274 | mlir::Location loc, mlir::Value resultBox, |
| 275 | mlir::Value arrayBox, mlir::Value shiftBox) { |
| 276 | auto cshiftFunc = |
| 277 | fir::runtime::getRuntimeFunc<mkRTKey(CshiftVector)>(loc, builder); |
| 278 | auto fTy = cshiftFunc.getFunctionType(); |
| 279 | |
| 280 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 281 | auto sourceLine = |
| 282 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(4)); |
| 283 | auto args = fir::runtime::createArguments( |
| 284 | builder, loc, fTy, resultBox, arrayBox, shiftBox, sourceFile, sourceLine); |
| 285 | builder.create<fir::CallOp>(loc, cshiftFunc, args); |
| 286 | } |
| 287 | |
| 288 | /// Generate call to Eoshift intrinsic |
| 289 | void fir::runtime::genEoshift(fir::FirOpBuilder &builder, mlir::Location loc, |
| 290 | mlir::Value resultBox, mlir::Value arrayBox, |
| 291 | mlir::Value shiftBox, mlir::Value boundBox, |
| 292 | mlir::Value dimBox) { |
| 293 | auto eoshiftFunc = |
| 294 | fir::runtime::getRuntimeFunc<mkRTKey(Eoshift)>(loc, builder); |
| 295 | auto fTy = eoshiftFunc.getFunctionType(); |
| 296 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 297 | auto sourceLine = |
| 298 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(6)); |
| 299 | auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox, |
| 300 | arrayBox, shiftBox, boundBox, |
| 301 | dimBox, sourceFile, sourceLine); |
| 302 | builder.create<fir::CallOp>(loc, eoshiftFunc, args); |
| 303 | } |
| 304 | |
| 305 | /// Generate call to the vector version of the Eoshift intrinsic |
| 306 | void fir::runtime::genEoshiftVector(fir::FirOpBuilder &builder, |
| 307 | mlir::Location loc, mlir::Value resultBox, |
| 308 | mlir::Value arrayBox, mlir::Value shiftBox, |
| 309 | mlir::Value boundBox) { |
| 310 | auto eoshiftFunc = |
| 311 | fir::runtime::getRuntimeFunc<mkRTKey(EoshiftVector)>(loc, builder); |
| 312 | auto fTy = eoshiftFunc.getFunctionType(); |
| 313 | |
| 314 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 315 | auto sourceLine = |
| 316 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(5)); |
| 317 | |
| 318 | auto args = |
| 319 | fir::runtime::createArguments(builder, loc, fTy, resultBox, arrayBox, |
| 320 | shiftBox, boundBox, sourceFile, sourceLine); |
| 321 | builder.create<fir::CallOp>(loc, eoshiftFunc, args); |
| 322 | } |
| 323 | |
| 324 | /// Define ForcedMatmul<ACAT><AKIND><BCAT><BKIND> models. |
| 325 | struct ForcedMatmulTypeModel { |
| 326 | static constexpr fir::runtime::FuncTypeBuilderFunc getTypeModel() { |
| 327 | return [](mlir::MLIRContext *ctx) { |
| 328 | auto boxRefTy = |
| 329 | fir::runtime::getModel<Fortran::runtime::Descriptor &>()(ctx); |
| 330 | auto boxTy = |
| 331 | fir::runtime::getModel<const Fortran::runtime::Descriptor &>()(ctx); |
| 332 | auto strTy = fir::runtime::getModel<const char *>()(ctx); |
| 333 | auto intTy = fir::runtime::getModel<int>()(ctx); |
| 334 | return mlir::FunctionType::get( |
| 335 | ctx, {boxRefTy, boxTy, boxTy, strTy, intTy}, {}); |
| 336 | }; |
| 337 | } |
| 338 | }; |
| 339 | |
| 340 | #define MATMUL_INSTANCE(ACAT, AKIND, BCAT, BKIND) \ |
| 341 | struct ForcedMatmul##ACAT##AKIND##BCAT##BKIND \ |
| 342 | : public ForcedMatmulTypeModel { \ |
| 343 | static constexpr const char *name = \ |
| 344 | ExpandAndQuoteKey(RTNAME(Matmul##ACAT##AKIND##BCAT##BKIND)); \ |
| 345 | }; |
| 346 | |
| 347 | #define MATMUL_DIRECT_INSTANCE(ACAT, AKIND, BCAT, BKIND) |
| 348 | #define MATMUL_FORCE_ALL_TYPES 1 |
| 349 | |
| 350 | #include "flang/Runtime/matmul-instances.inc" |
| 351 | |
| 352 | /// Generate call to Matmul intrinsic runtime routine. |
| 353 | void fir::runtime::genMatmul(fir::FirOpBuilder &builder, mlir::Location loc, |
| 354 | mlir::Value resultBox, mlir::Value matrixABox, |
| 355 | mlir::Value matrixBBox) { |
| 356 | mlir::func::FuncOp func; |
| 357 | auto boxATy = matrixABox.getType(); |
| 358 | auto arrATy = fir::dyn_cast_ptrOrBoxEleTy(boxATy); |
| 359 | auto arrAEleTy = mlir::cast<fir::SequenceType>(arrATy).getElementType(); |
| 360 | auto [aCat, aKind] = fir::mlirTypeToCategoryKind(loc, arrAEleTy); |
| 361 | auto boxBTy = matrixBBox.getType(); |
| 362 | auto arrBTy = fir::dyn_cast_ptrOrBoxEleTy(boxBTy); |
| 363 | auto arrBEleTy = mlir::cast<fir::SequenceType>(arrBTy).getElementType(); |
| 364 | auto [bCat, bKind] = fir::mlirTypeToCategoryKind(loc, arrBEleTy); |
| 365 | |
| 366 | // Unsigned is treated as Integer when both operands are unsigned/integer |
| 367 | #define MATMUL_INSTANCE(ACAT, AKIND, BCAT, BKIND) \ |
| 368 | if (!func && aKind == AKIND && bKind == BKIND && \ |
| 369 | ((aCat == TypeCategory::ACAT && bCat == TypeCategory::BCAT) || \ |
| 370 | ((aCat == TypeCategory::Integer || aCat == TypeCategory::Unsigned) && \ |
| 371 | (bCat == TypeCategory::Integer || bCat == TypeCategory::Unsigned)))) { \ |
| 372 | func = \ |
| 373 | fir::runtime::getRuntimeFunc<ForcedMatmul##ACAT##AKIND##BCAT##BKIND>( \ |
| 374 | loc, builder); \ |
| 375 | } |
| 376 | |
| 377 | #define MATMUL_DIRECT_INSTANCE(ACAT, AKIND, BCAT, BKIND) |
| 378 | #define MATMUL_FORCE_ALL_TYPES 1 |
| 379 | #include "flang/Runtime/matmul-instances.inc" |
| 380 | |
| 381 | if (!func) { |
| 382 | fir::intrinsicTypeTODO2(builder, arrAEleTy, arrBEleTy, loc, "MATMUL" ); |
| 383 | } |
| 384 | auto fTy = func.getFunctionType(); |
| 385 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 386 | auto sourceLine = |
| 387 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(4)); |
| 388 | auto args = |
| 389 | fir::runtime::createArguments(builder, loc, fTy, resultBox, matrixABox, |
| 390 | matrixBBox, sourceFile, sourceLine); |
| 391 | builder.create<fir::CallOp>(loc, func, args); |
| 392 | } |
| 393 | |
| 394 | /// Define ForcedMatmulTranspose<ACAT><AKIND><BCAT><BKIND> models. |
| 395 | #define MATMUL_INSTANCE(ACAT, AKIND, BCAT, BKIND) \ |
| 396 | struct ForcedMatmulTranspose##ACAT##AKIND##BCAT##BKIND \ |
| 397 | : public ForcedMatmulTypeModel { \ |
| 398 | static constexpr const char *name = \ |
| 399 | ExpandAndQuoteKey(RTNAME(MatmulTranspose##ACAT##AKIND##BCAT##BKIND)); \ |
| 400 | }; |
| 401 | |
| 402 | #define MATMUL_DIRECT_INSTANCE(ACAT, AKIND, BCAT, BKIND) |
| 403 | #define MATMUL_FORCE_ALL_TYPES 1 |
| 404 | |
| 405 | #include "flang/Runtime/matmul-instances.inc" |
| 406 | |
| 407 | void fir::runtime::genMatmulTranspose(fir::FirOpBuilder &builder, |
| 408 | mlir::Location loc, mlir::Value resultBox, |
| 409 | mlir::Value matrixABox, |
| 410 | mlir::Value matrixBBox) { |
| 411 | mlir::func::FuncOp func; |
| 412 | auto boxATy = matrixABox.getType(); |
| 413 | auto arrATy = fir::dyn_cast_ptrOrBoxEleTy(boxATy); |
| 414 | auto arrAEleTy = mlir::cast<fir::SequenceType>(arrATy).getElementType(); |
| 415 | auto [aCat, aKind] = fir::mlirTypeToCategoryKind(loc, arrAEleTy); |
| 416 | auto boxBTy = matrixBBox.getType(); |
| 417 | auto arrBTy = fir::dyn_cast_ptrOrBoxEleTy(boxBTy); |
| 418 | auto arrBEleTy = mlir::cast<fir::SequenceType>(arrBTy).getElementType(); |
| 419 | auto [bCat, bKind] = fir::mlirTypeToCategoryKind(loc, arrBEleTy); |
| 420 | |
| 421 | #define MATMUL_INSTANCE(ACAT, AKIND, BCAT, BKIND) \ |
| 422 | if (!func && aCat == TypeCategory::ACAT && aKind == AKIND && \ |
| 423 | bCat == TypeCategory::BCAT && bKind == BKIND) { \ |
| 424 | func = fir::runtime::getRuntimeFunc< \ |
| 425 | ForcedMatmulTranspose##ACAT##AKIND##BCAT##BKIND>(loc, builder); \ |
| 426 | } |
| 427 | |
| 428 | #define MATMUL_DIRECT_INSTANCE(ACAT, AKIND, BCAT, BKIND) |
| 429 | #define MATMUL_FORCE_ALL_TYPES 1 |
| 430 | #include "flang/Runtime/matmul-instances.inc" |
| 431 | |
| 432 | if (!func) { |
| 433 | fir::intrinsicTypeTODO2(builder, arrAEleTy, arrBEleTy, loc, |
| 434 | "MATMUL-TRANSPOSE" ); |
| 435 | } |
| 436 | auto fTy = func.getFunctionType(); |
| 437 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 438 | auto sourceLine = |
| 439 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(4)); |
| 440 | auto args = |
| 441 | fir::runtime::createArguments(builder, loc, fTy, resultBox, matrixABox, |
| 442 | matrixBBox, sourceFile, sourceLine); |
| 443 | builder.create<fir::CallOp>(loc, func, args); |
| 444 | } |
| 445 | |
| 446 | /// Generate call to Pack intrinsic runtime routine. |
| 447 | void fir::runtime::genPack(fir::FirOpBuilder &builder, mlir::Location loc, |
| 448 | mlir::Value resultBox, mlir::Value arrayBox, |
| 449 | mlir::Value maskBox, mlir::Value vectorBox) { |
| 450 | auto packFunc = fir::runtime::getRuntimeFunc<mkRTKey(Pack)>(loc, builder); |
| 451 | auto fTy = packFunc.getFunctionType(); |
| 452 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 453 | auto sourceLine = |
| 454 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(5)); |
| 455 | auto args = |
| 456 | fir::runtime::createArguments(builder, loc, fTy, resultBox, arrayBox, |
| 457 | maskBox, vectorBox, sourceFile, sourceLine); |
| 458 | builder.create<fir::CallOp>(loc, packFunc, args); |
| 459 | } |
| 460 | |
| 461 | /// Generate call to Reshape intrinsic runtime routine. |
| 462 | void fir::runtime::genReshape(fir::FirOpBuilder &builder, mlir::Location loc, |
| 463 | mlir::Value resultBox, mlir::Value sourceBox, |
| 464 | mlir::Value shapeBox, mlir::Value padBox, |
| 465 | mlir::Value orderBox) { |
| 466 | auto func = fir::runtime::getRuntimeFunc<mkRTKey(Reshape)>(loc, builder); |
| 467 | auto fTy = func.getFunctionType(); |
| 468 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 469 | auto sourceLine = |
| 470 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(6)); |
| 471 | auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox, |
| 472 | sourceBox, shapeBox, padBox, |
| 473 | orderBox, sourceFile, sourceLine); |
| 474 | builder.create<fir::CallOp>(loc, func, args); |
| 475 | } |
| 476 | |
| 477 | /// Generate call to ShallowCopy[Direct] runtime routine. |
| 478 | /// ShallowCopyDirect is used iff \p resultIsAllocated is true. |
| 479 | void fir::runtime::genShallowCopy(fir::FirOpBuilder &builder, |
| 480 | mlir::Location loc, mlir::Value resultBox, |
| 481 | mlir::Value arrayBox, |
| 482 | bool resultIsAllocated) { |
| 483 | auto packFunc = |
| 484 | resultIsAllocated |
| 485 | ? fir::runtime::getRuntimeFunc<mkRTKey(ShallowCopyDirect)>(loc, |
| 486 | builder) |
| 487 | : fir::runtime::getRuntimeFunc<mkRTKey(ShallowCopy)>(loc, builder); |
| 488 | auto fTy = packFunc.getFunctionType(); |
| 489 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 490 | auto sourceLine = |
| 491 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(3)); |
| 492 | auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox, |
| 493 | arrayBox, sourceFile, sourceLine); |
| 494 | builder.create<fir::CallOp>(loc, packFunc, args); |
| 495 | } |
| 496 | |
| 497 | /// Generate call to Spread intrinsic runtime routine. |
| 498 | void fir::runtime::genSpread(fir::FirOpBuilder &builder, mlir::Location loc, |
| 499 | mlir::Value resultBox, mlir::Value sourceBox, |
| 500 | mlir::Value dim, mlir::Value ncopies) { |
| 501 | auto func = fir::runtime::getRuntimeFunc<mkRTKey(Spread)>(loc, builder); |
| 502 | auto fTy = func.getFunctionType(); |
| 503 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 504 | auto sourceLine = |
| 505 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(5)); |
| 506 | auto args = |
| 507 | fir::runtime::createArguments(builder, loc, fTy, resultBox, sourceBox, |
| 508 | dim, ncopies, sourceFile, sourceLine); |
| 509 | builder.create<fir::CallOp>(loc, func, args); |
| 510 | } |
| 511 | |
| 512 | /// Generate call to Transpose intrinsic runtime routine. |
| 513 | void fir::runtime::genTranspose(fir::FirOpBuilder &builder, mlir::Location loc, |
| 514 | mlir::Value resultBox, mlir::Value sourceBox) { |
| 515 | auto func = fir::runtime::getRuntimeFunc<mkRTKey(Transpose)>(loc, builder); |
| 516 | auto fTy = func.getFunctionType(); |
| 517 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 518 | auto sourceLine = |
| 519 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(3)); |
| 520 | auto args = fir::runtime::createArguments(builder, loc, fTy, resultBox, |
| 521 | sourceBox, sourceFile, sourceLine); |
| 522 | builder.create<fir::CallOp>(loc, func, args); |
| 523 | } |
| 524 | |
| 525 | /// Generate call to Unpack intrinsic runtime routine. |
| 526 | void fir::runtime::genUnpack(fir::FirOpBuilder &builder, mlir::Location loc, |
| 527 | mlir::Value resultBox, mlir::Value vectorBox, |
| 528 | mlir::Value maskBox, mlir::Value fieldBox) { |
| 529 | auto unpackFunc = fir::runtime::getRuntimeFunc<mkRTKey(Unpack)>(loc, builder); |
| 530 | auto fTy = unpackFunc.getFunctionType(); |
| 531 | auto sourceFile = fir::factory::locationToFilename(builder, loc); |
| 532 | auto sourceLine = |
| 533 | fir::factory::locationToLineNo(builder, loc, fTy.getInput(5)); |
| 534 | auto args = |
| 535 | fir::runtime::createArguments(builder, loc, fTy, resultBox, vectorBox, |
| 536 | maskBox, fieldBox, sourceFile, sourceLine); |
| 537 | builder.create<fir::CallOp>(loc, unpackFunc, args); |
| 538 | } |
| 539 | |