1//===- OpDocGen.cpp - MLIR operation documentation generator --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// OpDocGen uses the description of operations to generate documentation for the
10// operations.
11//
12//===----------------------------------------------------------------------===//
13
14#include "DialectGenUtilities.h"
15#include "DocGenUtilities.h"
16#include "OpGenHelpers.h"
17#include "mlir/Support/IndentedOstream.h"
18#include "mlir/TableGen/AttrOrTypeDef.h"
19#include "mlir/TableGen/GenInfo.h"
20#include "mlir/TableGen/Operator.h"
21#include "llvm/ADT/DenseMap.h"
22#include "llvm/ADT/SetVector.h"
23#include "llvm/ADT/StringExtras.h"
24#include "llvm/ADT/StringRef.h"
25#include "llvm/Support/CommandLine.h"
26#include "llvm/Support/FormatVariadic.h"
27#include "llvm/Support/Regex.h"
28#include "llvm/Support/Signals.h"
29#include "llvm/TableGen/Error.h"
30#include "llvm/TableGen/Record.h"
31#include "llvm/TableGen/TableGenBackend.h"
32
33#include <set>
34#include <string>
35
36//===----------------------------------------------------------------------===//
37// Commandline Options
38//===----------------------------------------------------------------------===//
39static llvm::cl::OptionCategory
40 docCat("Options for -gen-(attrdef|typedef|op|dialect)-doc");
41llvm::cl::opt<std::string>
42 stripPrefix("strip-prefix",
43 llvm::cl::desc("Strip prefix of the fully qualified names"),
44 llvm::cl::init(Val: "::mlir::"), llvm::cl::cat(docCat));
45llvm::cl::opt<bool> allowHugoSpecificFeatures(
46 "allow-hugo-specific-features",
47 llvm::cl::desc("Allows using features specific to Hugo"),
48 llvm::cl::init(Val: false), llvm::cl::cat(docCat));
49
50using namespace llvm;
51using namespace mlir;
52using namespace mlir::tblgen;
53using mlir::tblgen::Operator;
54
55void mlir::tblgen::emitSummary(StringRef summary, raw_ostream &os) {
56 if (!summary.empty()) {
57 llvm::StringRef trimmed = summary.trim();
58 char first = std::toupper(c: trimmed.front());
59 llvm::StringRef rest = trimmed.drop_front();
60 os << "\n_" << first << rest << "_\n\n";
61 }
62}
63
64// Emit the description by aligning the text to the left per line (e.g.,
65// removing the minimum indentation across the block).
66//
67// This expects that the description in the tablegen file is already formatted
68// in a way the user wanted but has some additional indenting due to being
69// nested in the op definition.
70void mlir::tblgen::emitDescription(StringRef description, raw_ostream &os) {
71 raw_indented_ostream ros(os);
72 ros.printReindented(str: description.rtrim(Chars: " \t"));
73}
74
75void mlir::tblgen::emitDescriptionComment(StringRef description,
76 raw_ostream &os, StringRef prefix) {
77 if (description.empty())
78 return;
79 raw_indented_ostream ros(os);
80 StringRef trimmed = description.rtrim(Chars: " \t");
81 ros.printReindented(str: trimmed, extraPrefix: (Twine(prefix) + "/// ").str());
82 if (!trimmed.ends_with(Suffix: "\n"))
83 ros << "\n";
84}
85
86// Emits `str` with trailing newline if not empty.
87static void emitIfNotEmpty(StringRef str, raw_ostream &os) {
88 if (!str.empty()) {
89 emitDescription(description: str, os);
90 os << "\n";
91 }
92}
93
94/// Emit the given named constraint.
95template <typename T>
96static void emitNamedConstraint(const T &it, raw_ostream &os) {
97 if (!it.name.empty())
98 os << "| `" << it.name << "`";
99 else
100 os << "&laquo;unnamed&raquo;";
101 os << " | " << it.constraint.getSummary() << "\n";
102}
103
104//===----------------------------------------------------------------------===//
105// Operation Documentation
106//===----------------------------------------------------------------------===//
107
108/// Emit the assembly format of an operation.
109static void emitAssemblyFormat(StringRef opName, StringRef format,
110 raw_ostream &os) {
111 os << "\nSyntax:\n\n```\noperation ::= `" << opName << "` ";
112
113 // Print the assembly format aligned.
114 unsigned indent = strlen(s: "operation ::= ");
115 std::pair<StringRef, StringRef> split = format.split(Separator: '\n');
116 os << split.first.trim() << "\n";
117 do {
118 split = split.second.split(Separator: '\n');
119 StringRef formatChunk = split.first.trim();
120 if (!formatChunk.empty())
121 os.indent(NumSpaces: indent) << formatChunk << "\n";
122 } while (!split.second.empty());
123 os << "```\n\n";
124}
125
126/// Place `text` between backticks so that the Markdown processor renders it as
127/// inline code.
128static std::string backticks(const std::string &text) {
129 return '`' + text + '`';
130}
131
132static void emitOpTraitsDoc(const Operator &op, raw_ostream &os) {
133 // TODO: We should link to the trait/documentation of it. That also means we
134 // should add descriptions to traits that can be queried.
135 // Collect using set to sort effects, interfaces & traits.
136 std::set<std::string> effects, interfaces, traits;
137 for (auto &trait : op.getTraits()) {
138 if (isa<PredTrait>(Val: &trait))
139 continue;
140
141 std::string name = trait.getDef().getName().str();
142 StringRef ref = name;
143 StringRef traitName = trait.getDef().getValueAsString(FieldName: "trait");
144 traitName.consume_back(Suffix: "::Trait");
145 traitName.consume_back(Suffix: "::Impl");
146 if (ref.starts_with(Prefix: "anonymous_"))
147 name = traitName.str();
148 if (isa<InterfaceTrait>(Val: &trait)) {
149 if (trait.getDef().isSubClassOf(Name: "SideEffectsTraitBase")) {
150 auto effectName = trait.getDef().getValueAsString(FieldName: "baseEffectName");
151 effectName.consume_front(Prefix: "::");
152 effectName.consume_front(Prefix: "mlir::");
153 std::string effectStr;
154 llvm::raw_string_ostream os(effectStr);
155 os << effectName << "{";
156 auto list = trait.getDef().getValueAsListOfDefs(FieldName: "effects");
157 llvm::interleaveComma(c: list, os, each_fn: [&](Record *rec) {
158 StringRef effect = rec->getValueAsString(FieldName: "effect");
159 effect.consume_front(Prefix: "::");
160 effect.consume_front(Prefix: "mlir::");
161 os << effect << " on " << rec->getValueAsString(FieldName: "resource");
162 });
163 os << "}";
164 effects.insert(x: backticks(text: os.str()));
165 name.append(str: llvm::formatv(Fmt: " ({0})", Vals&: traitName).str());
166 }
167 interfaces.insert(x: backticks(text: name));
168 continue;
169 }
170
171 traits.insert(x: backticks(text: name));
172 }
173 if (!traits.empty()) {
174 llvm::interleaveComma(c: traits, os&: os << "\nTraits: ");
175 os << "\n";
176 }
177 if (!interfaces.empty()) {
178 llvm::interleaveComma(c: interfaces, os&: os << "\nInterfaces: ");
179 os << "\n";
180 }
181 if (!effects.empty()) {
182 llvm::interleaveComma(c: effects, os&: os << "\nEffects: ");
183 os << "\n";
184 }
185}
186
187static StringRef resolveAttrDescription(const Attribute &attr) {
188 StringRef description = attr.getDescription();
189 if (description.empty())
190 return attr.getBaseAttr().getDescription();
191 return description;
192}
193
194static void emitOpDoc(const Operator &op, raw_ostream &os) {
195 std::string classNameStr = op.getQualCppClassName();
196 StringRef className = classNameStr;
197 (void)className.consume_front(Prefix: stripPrefix);
198 os << llvm::formatv(Fmt: "### `{0}` ({1})\n", Vals: op.getOperationName(), Vals&: className);
199
200 // Emit the summary, syntax, and description if present.
201 if (op.hasSummary())
202 emitSummary(summary: op.getSummary(), os);
203 if (op.hasAssemblyFormat())
204 emitAssemblyFormat(opName: op.getOperationName(), format: op.getAssemblyFormat().trim(),
205 os);
206 if (op.hasDescription())
207 mlir::tblgen::emitDescription(description: op.getDescription(), os);
208
209 emitOpTraitsDoc(op, os);
210
211 // Emit attributes.
212 if (op.getNumAttributes() != 0) {
213 os << "\n#### Attributes:\n\n";
214 // Note: This table is HTML rather than markdown so the attribute's
215 // description can appear in an expandable region. The description may be
216 // multiple lines, which is not supported in a markdown table cell.
217 os << "<table>\n";
218 // Header.
219 os << "<tr><th>Attribute</th><th>MLIR Type</th><th>Description</th></tr>\n";
220 for (const auto &it : op.getAttributes()) {
221 StringRef storageType = it.attr.getStorageType();
222 // Name and storage type.
223 os << "<tr>";
224 os << "<td><code>" << it.name << "</code></td><td>" << storageType
225 << "</td><td>";
226 StringRef description = resolveAttrDescription(attr: it.attr);
227 if (allowHugoSpecificFeatures && !description.empty()) {
228 // Expandable description.
229 // This appears as just the summary, but when clicked shows the full
230 // description.
231 os << "<details>"
232 << "<summary>" << it.attr.getSummary() << "</summary>"
233 << "{{% markdown %}}" << description << "{{% /markdown %}}"
234 << "</details>";
235 } else {
236 // Fallback: Single-line summary.
237 os << it.attr.getSummary();
238 }
239 os << "</td></tr>\n";
240 }
241 os << "</table>\n";
242 }
243
244 // Emit each of the operands.
245 if (op.getNumOperands() != 0) {
246 os << "\n#### Operands:\n\n";
247 os << "| Operand | Description |\n"
248 << "| :-----: | ----------- |\n";
249 for (const auto &it : op.getOperands())
250 emitNamedConstraint(it, os);
251 }
252
253 // Emit results.
254 if (op.getNumResults() != 0) {
255 os << "\n#### Results:\n\n";
256 os << "| Result | Description |\n"
257 << "| :----: | ----------- |\n";
258 for (const auto &it : op.getResults())
259 emitNamedConstraint(it, os);
260 }
261
262 // Emit successors.
263 if (op.getNumSuccessors() != 0) {
264 os << "\n#### Successors:\n\n";
265 os << "| Successor | Description |\n"
266 << "| :-------: | ----------- |\n";
267 for (const auto &it : op.getSuccessors())
268 emitNamedConstraint(it, os);
269 }
270
271 os << "\n";
272}
273
274static void emitSourceLink(StringRef inputFilename, raw_ostream &os) {
275 size_t pathBegin = inputFilename.find(Str: "mlir/include/mlir/");
276 if (pathBegin == StringRef::npos)
277 return;
278
279 StringRef inputFromMlirInclude = inputFilename.substr(Start: pathBegin);
280
281 os << "[source](https://github.com/llvm/llvm-project/blob/main/"
282 << inputFromMlirInclude << ")\n\n";
283}
284
285static void emitOpDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
286 auto opDefs = getRequestedOpDefinitions(recordKeeper);
287
288 os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
289 emitSourceLink(inputFilename: recordKeeper.getInputFilename(), os);
290 for (const llvm::Record *opDef : opDefs)
291 emitOpDoc(op: Operator(opDef), os);
292}
293
294//===----------------------------------------------------------------------===//
295// Attribute Documentation
296//===----------------------------------------------------------------------===//
297
298static void emitAttrDoc(const Attribute &attr, raw_ostream &os) {
299 os << "### " << attr.getSummary() << "\n\n";
300 emitDescription(description: attr.getDescription(), os);
301 os << "\n\n";
302}
303
304//===----------------------------------------------------------------------===//
305// Type Documentation
306//===----------------------------------------------------------------------===//
307
308static void emitTypeDoc(const Type &type, raw_ostream &os) {
309 os << "### " << type.getSummary() << "\n\n";
310 emitDescription(description: type.getDescription(), os);
311 os << "\n\n";
312}
313
314//===----------------------------------------------------------------------===//
315// TypeDef Documentation
316//===----------------------------------------------------------------------===//
317
318static void emitAttrOrTypeDefAssemblyFormat(const AttrOrTypeDef &def,
319 raw_ostream &os) {
320 ArrayRef<AttrOrTypeParameter> parameters = def.getParameters();
321 char prefix = isa<AttrDef>(Val: def) ? '#' : '!';
322 if (parameters.empty()) {
323 os << "\nSyntax: `" << prefix << def.getDialect().getName() << "."
324 << def.getMnemonic() << "`\n";
325 return;
326 }
327
328 os << "\nSyntax:\n\n```\n"
329 << prefix << def.getDialect().getName() << "." << def.getMnemonic()
330 << "<\n";
331 for (const auto &it : llvm::enumerate(First&: parameters)) {
332 const AttrOrTypeParameter &param = it.value();
333 os << " " << param.getSyntax();
334 if (it.index() < (parameters.size() - 1))
335 os << ",";
336 os << " # " << param.getName() << "\n";
337 }
338 os << ">\n```\n";
339}
340
341static void emitAttrOrTypeDefDoc(const AttrOrTypeDef &def, raw_ostream &os) {
342 os << llvm::formatv(Fmt: "### {0}\n", Vals: def.getCppClassName());
343
344 // Emit the summary if present.
345 if (def.hasSummary())
346 os << "\n" << def.getSummary() << "\n";
347
348 // Emit the syntax if present.
349 if (def.getMnemonic() && !def.hasCustomAssemblyFormat())
350 emitAttrOrTypeDefAssemblyFormat(def, os);
351
352 // Emit the description if present.
353 if (def.hasDescription()) {
354 os << "\n";
355 mlir::tblgen::emitDescription(description: def.getDescription(), os);
356 }
357
358 // Emit parameter documentation.
359 ArrayRef<AttrOrTypeParameter> parameters = def.getParameters();
360 if (!parameters.empty()) {
361 os << "\n#### Parameters:\n\n";
362 os << "| Parameter | C++ type | Description |\n"
363 << "| :-------: | :-------: | ----------- |\n";
364 for (const auto &it : parameters) {
365 auto desc = it.getSummary();
366 os << "| " << it.getName() << " | `" << it.getCppType() << "` | "
367 << (desc ? *desc : "") << " |\n";
368 }
369 }
370
371 os << "\n";
372}
373
374static void emitAttrOrTypeDefDoc(const RecordKeeper &recordKeeper,
375 raw_ostream &os, StringRef recordTypeName) {
376 std::vector<llvm::Record *> defs =
377 recordKeeper.getAllDerivedDefinitions(ClassName: recordTypeName);
378
379 os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
380 for (const llvm::Record *def : defs)
381 emitAttrOrTypeDefDoc(def: AttrOrTypeDef(def), os);
382}
383
384//===----------------------------------------------------------------------===//
385// Dialect Documentation
386//===----------------------------------------------------------------------===//
387
388struct OpDocGroup {
389 const Dialect &getDialect() const { return ops.front().getDialect(); }
390
391 // Returns the summary description of the section.
392 std::string summary = "";
393
394 // Returns the description of the section.
395 StringRef description = "";
396
397 // Instances inside the section.
398 std::vector<Operator> ops;
399};
400
401static void maybeNest(bool nest, llvm::function_ref<void(raw_ostream &os)> fn,
402 raw_ostream &os) {
403 std::string str;
404 llvm::raw_string_ostream ss(str);
405 fn(ss);
406 for (StringRef x : llvm::split(Str: ss.str(), Separator: "\n")) {
407 if (nest && x.starts_with(Prefix: "#"))
408 os << "#";
409 os << x << "\n";
410 }
411}
412
413static void emitBlock(ArrayRef<Attribute> attributes, StringRef inputFilename,
414 ArrayRef<AttrDef> attrDefs, ArrayRef<OpDocGroup> ops,
415 ArrayRef<Type> types, ArrayRef<TypeDef> typeDefs,
416 raw_ostream &os) {
417 if (!ops.empty()) {
418 os << "## Operations\n\n";
419 emitSourceLink(inputFilename, os);
420 for (const OpDocGroup &grouping : ops) {
421 bool nested = !grouping.summary.empty();
422 maybeNest(
423 nest: nested,
424 fn: [&](raw_ostream &os) {
425 if (nested) {
426 os << "## " << StringRef(grouping.summary).trim() << "\n\n";
427 emitDescription(description: grouping.description, os);
428 os << "\n\n";
429 }
430 for (const Operator &op : grouping.ops) {
431 emitOpDoc(op, os);
432 }
433 },
434 os);
435 }
436 }
437
438 if (!attributes.empty()) {
439 os << "## Attribute constraints\n\n";
440 for (const Attribute &attr : attributes)
441 emitAttrDoc(attr, os);
442 }
443
444 if (!attrDefs.empty()) {
445 os << "## Attributes\n\n";
446 for (const AttrDef &def : attrDefs)
447 emitAttrOrTypeDefDoc(def, os);
448 }
449
450 // TODO: Add link between use and def for types
451 if (!types.empty()) {
452 os << "## Type constraints\n\n";
453 for (const Type &type : types)
454 emitTypeDoc(type, os);
455 }
456
457 if (!typeDefs.empty()) {
458 os << "## Types\n\n";
459 for (const TypeDef &def : typeDefs)
460 emitAttrOrTypeDefDoc(def, os);
461 }
462}
463
464static void emitDialectDoc(const Dialect &dialect, StringRef inputFilename,
465 ArrayRef<Attribute> attributes,
466 ArrayRef<AttrDef> attrDefs, ArrayRef<OpDocGroup> ops,
467 ArrayRef<Type> types, ArrayRef<TypeDef> typeDefs,
468 raw_ostream &os) {
469 os << "# '" << dialect.getName() << "' Dialect\n\n";
470 emitIfNotEmpty(str: dialect.getSummary(), os);
471 emitIfNotEmpty(str: dialect.getDescription(), os);
472
473 // Generate a TOC marker except if description already contains one.
474 llvm::Regex r("^[[:space:]]*\\[TOC\\]$", llvm::Regex::RegexFlags::Newline);
475 if (!r.match(String: dialect.getDescription()))
476 os << "[TOC]\n\n";
477
478 emitBlock(attributes, inputFilename, attrDefs, ops, types, typeDefs, os);
479}
480
481static bool emitDialectDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
482 std::vector<Record *> dialectDefs =
483 recordKeeper.getAllDerivedDefinitionsIfDefined(ClassName: "Dialect");
484 SmallVector<Dialect> dialects(dialectDefs.begin(), dialectDefs.end());
485 std::optional<Dialect> dialect = findDialectToGenerate(dialects);
486 if (!dialect)
487 return true;
488
489 std::vector<Record *> opDefs = getRequestedOpDefinitions(recordKeeper);
490 std::vector<Record *> attrDefs =
491 recordKeeper.getAllDerivedDefinitionsIfDefined(ClassName: "DialectAttr");
492 std::vector<Record *> typeDefs =
493 recordKeeper.getAllDerivedDefinitionsIfDefined(ClassName: "DialectType");
494 std::vector<Record *> typeDefDefs =
495 recordKeeper.getAllDerivedDefinitionsIfDefined(ClassName: "TypeDef");
496 std::vector<Record *> attrDefDefs =
497 recordKeeper.getAllDerivedDefinitionsIfDefined(ClassName: "AttrDef");
498
499 std::vector<Attribute> dialectAttrs;
500 std::vector<AttrDef> dialectAttrDefs;
501 std::vector<OpDocGroup> dialectOps;
502 std::vector<Type> dialectTypes;
503 std::vector<TypeDef> dialectTypeDefs;
504
505 llvm::SmallDenseSet<Record *> seen;
506 auto addIfInDialect = [&](llvm::Record *record, const auto &def, auto &vec) {
507 if (seen.insert(V: record).second && def.getDialect() == *dialect) {
508 vec.push_back(def);
509 return true;
510 }
511 return false;
512 };
513
514 SmallDenseMap<Record *, OpDocGroup> opDocGroup;
515
516 for (Record *def : attrDefDefs)
517 addIfInDialect(def, AttrDef(def), dialectAttrDefs);
518 for (Record *def : attrDefs)
519 addIfInDialect(def, Attribute(def), dialectAttrs);
520 for (Record *def : opDefs) {
521 if (Record *group = def->getValueAsOptionalDef(FieldName: "opDocGroup")) {
522 OpDocGroup &op = opDocGroup[group];
523 addIfInDialect(def, Operator(def), op.ops);
524 } else {
525 OpDocGroup op;
526 op.ops.emplace_back(args&: def);
527 addIfInDialect(def, op, dialectOps);
528 }
529 }
530 for (Record *rec :
531 recordKeeper.getAllDerivedDefinitionsIfDefined(ClassName: "OpDocGroup")) {
532 if (opDocGroup[rec].ops.empty())
533 continue;
534 opDocGroup[rec].summary = rec->getValueAsString(FieldName: "summary");
535 opDocGroup[rec].description = rec->getValueAsString(FieldName: "description");
536 dialectOps.push_back(x: opDocGroup[rec]);
537 }
538 for (Record *def : typeDefDefs)
539 addIfInDialect(def, TypeDef(def), dialectTypeDefs);
540 for (Record *def : typeDefs)
541 addIfInDialect(def, Type(def), dialectTypes);
542
543 // Sort alphabetically ignorning dialect for ops and section name for
544 // sections.
545 // TODO: The sorting order could be revised, currently attempting to sort of
546 // keep in alphabetical order.
547 std::sort(first: dialectOps.begin(), last: dialectOps.end(),
548 comp: [](const OpDocGroup &lhs, const OpDocGroup &rhs) {
549 auto getDesc = [](const OpDocGroup &arg) -> StringRef {
550 if (!arg.summary.empty())
551 return arg.summary;
552 return arg.ops.front().getDef().getValueAsString(FieldName: "opName");
553 };
554 return getDesc(lhs).compare_insensitive(RHS: getDesc(rhs)) < 0;
555 });
556
557 os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
558 emitDialectDoc(dialect: *dialect, inputFilename: recordKeeper.getInputFilename(), attributes: dialectAttrs,
559 attrDefs: dialectAttrDefs, ops: dialectOps, types: dialectTypes, typeDefs: dialectTypeDefs,
560 os);
561 return false;
562}
563
564//===----------------------------------------------------------------------===//
565// Gen Registration
566//===----------------------------------------------------------------------===//
567
568static mlir::GenRegistration
569 genAttrRegister("gen-attrdef-doc",
570 "Generate dialect attribute documentation",
571 [](const RecordKeeper &records, raw_ostream &os) {
572 emitAttrOrTypeDefDoc(recordKeeper: records, os, recordTypeName: "AttrDef");
573 return false;
574 });
575
576static mlir::GenRegistration
577 genOpRegister("gen-op-doc", "Generate dialect documentation",
578 [](const RecordKeeper &records, raw_ostream &os) {
579 emitOpDoc(recordKeeper: records, os);
580 return false;
581 });
582
583static mlir::GenRegistration
584 genTypeRegister("gen-typedef-doc", "Generate dialect type documentation",
585 [](const RecordKeeper &records, raw_ostream &os) {
586 emitAttrOrTypeDefDoc(recordKeeper: records, os, recordTypeName: "TypeDef");
587 return false;
588 });
589
590static mlir::GenRegistration
591 genRegister("gen-dialect-doc", "Generate dialect documentation",
592 [](const RecordKeeper &records, raw_ostream &os) {
593 return emitDialectDoc(recordKeeper: records, os);
594 });
595

source code of mlir/tools/mlir-tblgen/OpDocGen.cpp