1//===-- IR/VPIntrinsics.def - Describes llvm.vp.* Intrinsics -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains descriptions of the various Vector Predication intrinsics.
10// This is used as a central place for enumerating the different instructions
11// and should eventually be the place to put comments about the instructions.
12//
13//===----------------------------------------------------------------------===//
14
15// NOTE: NO INCLUDE GUARD DESIRED!
16
17// Provide definitions of macros so that users of this file do not have to
18// define everything to use it...
19//
20// Register a VP intrinsic and begin its property scope.
21// All VP intrinsic scopes are top level, ie it is illegal to place a
22// BEGIN_REGISTER_VP_INTRINSIC within a VP intrinsic scope.
23// \p VPID The VP intrinsic id.
24// \p MASKPOS The mask operand position.
25// \p EVLPOS The explicit vector length operand position.
26#ifndef BEGIN_REGISTER_VP_INTRINSIC
27#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS)
28#endif
29
30// End the property scope of a VP intrinsic.
31#ifndef END_REGISTER_VP_INTRINSIC
32#define END_REGISTER_VP_INTRINSIC(VPID)
33#endif
34
35// Register a new VP SDNode and begin its property scope.
36// When the SDNode scope is nested within a VP intrinsic scope, it is
37// implicitly registered as the canonical SDNode for this VP intrinsic. There
38// is one VP intrinsic that maps directly to one SDNode that goes by the
39// same name. Since the operands are also the same, we open the property
40// scopes for both the VPIntrinsic and the SDNode at once.
41// \p VPSD The SelectionDAG Node id (eg VP_ADD).
42// \p LEGALPOS The operand position of the SDNode that is used for legalizing.
43// If LEGALPOS < 0, then the return type given by
44// TheNode->getValueType(-1-LEGALPOS) is used.
45// \p TDNAME The name of the TableGen definition of this SDNode.
46// \p MASKPOS The mask operand position.
47// \p EVLPOS The explicit vector length operand position.
48#ifndef BEGIN_REGISTER_VP_SDNODE
49#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS)
50#endif
51
52// End the property scope of a new VP SDNode.
53#ifndef END_REGISTER_VP_SDNODE
54#define END_REGISTER_VP_SDNODE(VPSD)
55#endif
56
57// Helper macro to set up the mapping from VP intrinsic to ISD opcode.
58// Note: More than one VP intrinsic may map to one ISD opcode.
59#ifndef HELPER_MAP_VPID_TO_VPSD
60#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD)
61#endif
62
63// Helper macros for the common "1:1 - Intrinsic : SDNode" case.
64//
65// There is one VP intrinsic that maps directly to one SDNode that goes by the
66// same name. Since the operands are also the same, we open the property
67// scopes for both the VPIntrinsic and the SDNode at once.
68//
69// \p VPID The canonical name (eg `vp_add`, which at the same time is the
70// name of the intrinsic and the TableGen def of the SDNode).
71// \p MASKPOS The mask operand position.
72// \p EVLPOS The explicit vector length operand position.
73// \p VPSD The SelectionDAG Node id (eg VP_ADD).
74// \p LEGALPOS The operand position of the SDNode that is used for legalizing
75// this SDNode. This can be `-1`, in which case the return type of
76// the SDNode is used.
77#define BEGIN_REGISTER_VP(VPID, MASKPOS, EVLPOS, VPSD, LEGALPOS) \
78 BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS) \
79 BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, VPID, MASKPOS, EVLPOS) \
80 HELPER_MAP_VPID_TO_VPSD(VPID, VPSD)
81
82#define END_REGISTER_VP(VPID, VPSD) \
83 END_REGISTER_VP_INTRINSIC(VPID) \
84 END_REGISTER_VP_SDNODE(VPSD)
85
86// The following macros attach properties to the scope they are placed in. This
87// assigns the property to the VP Intrinsic and/or SDNode that belongs to the
88// scope.
89//
90// Property Macros {
91
92// The intrinsic and/or SDNode has the same function as this LLVM IR Opcode.
93// \p OPC The opcode of the instruction with the same function.
94#ifndef VP_PROPERTY_FUNCTIONAL_OPC
95#define VP_PROPERTY_FUNCTIONAL_OPC(OPC)
96#endif
97
98// Whether the intrinsic may have a rounding mode or exception behavior operand
99// bundle.
100// \p HASROUND '1' if the intrinsic can have a rounding mode operand bundle,
101// '0' otherwise.
102// \p HASEXCEPT '1' if the intrinsic can have an exception behavior operand
103// bundle, '0' otherwise.
104// \p INTRINID The constrained fp intrinsic this VP intrinsic corresponds to.
105#ifndef VP_PROPERTY_CONSTRAINEDFP
106#define VP_PROPERTY_CONSTRAINEDFP(HASROUND, HASEXCEPT, INTRINID)
107#endif
108
109// The intrinsic and/or SDNode has the same function as this ISD Opcode.
110// \p SDOPC The opcode of the instruction with the same function.
111#ifndef VP_PROPERTY_FUNCTIONAL_SDOPC
112#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC)
113#endif
114
115// Map this VP intrinsic to its canonical functional intrinsic.
116// \p INTRIN The non-VP intrinsics with the same function.
117#ifndef VP_PROPERTY_FUNCTIONAL_INTRINSIC
118#define VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN)
119#endif
120
121// This VP Intrinsic has no functionally-equivalent non-VP opcode or intrinsic.
122#ifndef VP_PROPERTY_NO_FUNCTIONAL
123#define VP_PROPERTY_NO_FUNCTIONAL
124#endif
125
126// This VP Intrinsic is a memory operation
127// The pointer arg is at POINTERPOS and the data arg is at DATAPOS.
128#ifndef VP_PROPERTY_MEMOP
129#define VP_PROPERTY_MEMOP(POINTERPOS, DATAPOS)
130#endif
131
132// Map this VP reduction intrinsic to its reduction operand positions.
133#ifndef VP_PROPERTY_REDUCTION
134#define VP_PROPERTY_REDUCTION(STARTPOS, VECTORPOS)
135#endif
136
137// A property to infer VP binary-op SDNode opcodes automatically.
138#ifndef VP_PROPERTY_BINARYOP
139#define VP_PROPERTY_BINARYOP
140#endif
141
142// A property to infer VP type casts automatically.
143#ifndef VP_PROPERTY_CASTOP
144#define VP_PROPERTY_CASTOP
145#endif
146
147// This VP Intrinsic is a comparison operation
148// The condition code arg is at CCPOS and accepts floating-point condition
149// codes if ISFP is set, else it accepts integer condition codes.
150#ifndef VP_PROPERTY_CMP
151#define VP_PROPERTY_CMP(CCPOS, ISFP)
152#endif
153
154/// } Property Macros
155
156///// Integer Arithmetic {
157
158// Specialized helper macro for integer binary operators (%x, %y, %mask, %evl).
159#ifdef HELPER_REGISTER_BINARY_INT_VP
160#error \
161 "The internal helper macro HELPER_REGISTER_BINARY_INT_VP is already defined!"
162#endif
163#define HELPER_REGISTER_BINARY_INT_VP(VPID, VPSD, IROPC, SDOPC) \
164 BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, -1) \
165 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \
166 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \
167 VP_PROPERTY_BINARYOP \
168 END_REGISTER_VP(VPID, VPSD)
169
170// llvm.vp.add(x,y,mask,vlen)
171HELPER_REGISTER_BINARY_INT_VP(vp_add, VP_ADD, Add, ADD)
172
173// llvm.vp.and(x,y,mask,vlen)
174HELPER_REGISTER_BINARY_INT_VP(vp_and, VP_AND, And, AND)
175
176// llvm.vp.ashr(x,y,mask,vlen)
177HELPER_REGISTER_BINARY_INT_VP(vp_ashr, VP_ASHR, AShr, SRA)
178
179// llvm.vp.lshr(x,y,mask,vlen)
180HELPER_REGISTER_BINARY_INT_VP(vp_lshr, VP_LSHR, LShr, SRL)
181
182// llvm.vp.mul(x,y,mask,vlen)
183HELPER_REGISTER_BINARY_INT_VP(vp_mul, VP_MUL, Mul, MUL)
184
185// llvm.vp.or(x,y,mask,vlen)
186HELPER_REGISTER_BINARY_INT_VP(vp_or, VP_OR, Or, OR)
187
188// llvm.vp.sdiv(x,y,mask,vlen)
189HELPER_REGISTER_BINARY_INT_VP(vp_sdiv, VP_SDIV, SDiv, SDIV)
190
191// llvm.vp.shl(x,y,mask,vlen)
192HELPER_REGISTER_BINARY_INT_VP(vp_shl, VP_SHL, Shl, SHL)
193
194// llvm.vp.srem(x,y,mask,vlen)
195HELPER_REGISTER_BINARY_INT_VP(vp_srem, VP_SREM, SRem, SREM)
196
197// llvm.vp.sub(x,y,mask,vlen)
198HELPER_REGISTER_BINARY_INT_VP(vp_sub, VP_SUB, Sub, SUB)
199
200// llvm.vp.udiv(x,y,mask,vlen)
201HELPER_REGISTER_BINARY_INT_VP(vp_udiv, VP_UDIV, UDiv, UDIV)
202
203// llvm.vp.urem(x,y,mask,vlen)
204HELPER_REGISTER_BINARY_INT_VP(vp_urem, VP_UREM, URem, UREM)
205
206// llvm.vp.xor(x,y,mask,vlen)
207HELPER_REGISTER_BINARY_INT_VP(vp_xor, VP_XOR, Xor, XOR)
208
209#undef HELPER_REGISTER_BINARY_INT_VP
210
211// llvm.vp.smin(x,y,mask,vlen)
212BEGIN_REGISTER_VP(vp_smin, 2, 3, VP_SMIN, -1)
213VP_PROPERTY_BINARYOP
214VP_PROPERTY_FUNCTIONAL_SDOPC(SMIN)
215VP_PROPERTY_FUNCTIONAL_INTRINSIC(smin)
216END_REGISTER_VP(vp_smin, VP_SMIN)
217
218// llvm.vp.smax(x,y,mask,vlen)
219BEGIN_REGISTER_VP(vp_smax, 2, 3, VP_SMAX, -1)
220VP_PROPERTY_BINARYOP
221VP_PROPERTY_FUNCTIONAL_SDOPC(SMAX)
222VP_PROPERTY_FUNCTIONAL_INTRINSIC(smax)
223END_REGISTER_VP(vp_smax, VP_SMAX)
224
225// llvm.vp.umin(x,y,mask,vlen)
226BEGIN_REGISTER_VP(vp_umin, 2, 3, VP_UMIN, -1)
227VP_PROPERTY_BINARYOP
228VP_PROPERTY_FUNCTIONAL_SDOPC(UMIN)
229VP_PROPERTY_FUNCTIONAL_INTRINSIC(umin)
230END_REGISTER_VP(vp_umin, VP_UMIN)
231
232// llvm.vp.umax(x,y,mask,vlen)
233BEGIN_REGISTER_VP(vp_umax, 2, 3, VP_UMAX, -1)
234VP_PROPERTY_BINARYOP
235VP_PROPERTY_FUNCTIONAL_SDOPC(UMAX)
236VP_PROPERTY_FUNCTIONAL_INTRINSIC(umax)
237END_REGISTER_VP(vp_umax, VP_UMAX)
238
239// llvm.vp.abs(x,is_int_min_poison,mask,vlen)
240BEGIN_REGISTER_VP_INTRINSIC(vp_abs, 2, 3)
241BEGIN_REGISTER_VP_SDNODE(VP_ABS, -1, vp_abs, 1, 2)
242HELPER_MAP_VPID_TO_VPSD(vp_abs, VP_ABS)
243VP_PROPERTY_FUNCTIONAL_INTRINSIC(abs)
244VP_PROPERTY_FUNCTIONAL_SDOPC(ABS)
245END_REGISTER_VP(vp_abs, VP_ABS)
246
247// llvm.vp.bswap(x,mask,vlen)
248BEGIN_REGISTER_VP(vp_bswap, 1, 2, VP_BSWAP, -1)
249VP_PROPERTY_FUNCTIONAL_INTRINSIC(bswap)
250VP_PROPERTY_FUNCTIONAL_SDOPC(BSWAP)
251END_REGISTER_VP(vp_bswap, VP_BSWAP)
252
253// llvm.vp.bitreverse(x,mask,vlen)
254BEGIN_REGISTER_VP(vp_bitreverse, 1, 2, VP_BITREVERSE, -1)
255VP_PROPERTY_FUNCTIONAL_INTRINSIC(bitreverse)
256VP_PROPERTY_FUNCTIONAL_SDOPC(BITREVERSE)
257END_REGISTER_VP(vp_bitreverse, VP_BITREVERSE)
258
259// llvm.vp.ctpop(x,mask,vlen)
260BEGIN_REGISTER_VP(vp_ctpop, 1, 2, VP_CTPOP, -1)
261VP_PROPERTY_FUNCTIONAL_INTRINSIC(ctpop)
262VP_PROPERTY_FUNCTIONAL_SDOPC(CTPOP)
263END_REGISTER_VP(vp_ctpop, VP_CTPOP)
264
265// llvm.vp.ctlz(x,is_zero_poison,mask,vlen)
266BEGIN_REGISTER_VP_INTRINSIC(vp_ctlz, 2, 3)
267BEGIN_REGISTER_VP_SDNODE(VP_CTLZ, -1, vp_ctlz, 1, 2)
268VP_PROPERTY_FUNCTIONAL_INTRINSIC(ctlz)
269VP_PROPERTY_FUNCTIONAL_SDOPC(CTLZ)
270END_REGISTER_VP_SDNODE(VP_CTLZ)
271BEGIN_REGISTER_VP_SDNODE(VP_CTLZ_ZERO_UNDEF, -1, vp_ctlz_zero_undef, 1, 2)
272END_REGISTER_VP_SDNODE(VP_CTLZ_ZERO_UNDEF)
273END_REGISTER_VP_INTRINSIC(vp_ctlz)
274
275// llvm.vp.cttz(x,is_zero_poison,mask,vlen)
276BEGIN_REGISTER_VP_INTRINSIC(vp_cttz, 2, 3)
277BEGIN_REGISTER_VP_SDNODE(VP_CTTZ, -1, vp_cttz, 1, 2)
278VP_PROPERTY_FUNCTIONAL_INTRINSIC(cttz)
279VP_PROPERTY_FUNCTIONAL_SDOPC(CTTZ)
280END_REGISTER_VP_SDNODE(VP_CTTZ)
281BEGIN_REGISTER_VP_SDNODE(VP_CTTZ_ZERO_UNDEF, -1, vp_cttz_zero_undef, 1, 2)
282END_REGISTER_VP_SDNODE(VP_CTTZ_ZERO_UNDEF)
283END_REGISTER_VP_INTRINSIC(vp_cttz)
284
285// llvm.vp.fshl(x,y,z,mask,vlen)
286BEGIN_REGISTER_VP(vp_fshl, 3, 4, VP_FSHL, -1)
287VP_PROPERTY_FUNCTIONAL_INTRINSIC(fshl)
288VP_PROPERTY_FUNCTIONAL_SDOPC(FSHL)
289END_REGISTER_VP(vp_fshl, VP_FSHL)
290
291// llvm.vp.fshr(x,y,z,mask,vlen)
292BEGIN_REGISTER_VP(vp_fshr, 3, 4, VP_FSHR, -1)
293VP_PROPERTY_FUNCTIONAL_INTRINSIC(fshr)
294VP_PROPERTY_FUNCTIONAL_SDOPC(FSHR)
295END_REGISTER_VP(vp_fshr, VP_FSHR)
296///// } Integer Arithmetic
297
298///// Floating-Point Arithmetic {
299
300// Specialized helper macro for floating-point binary operators
301// <operation>(%x, %y, %mask, %evl).
302#ifdef HELPER_REGISTER_BINARY_FP_VP
303#error \
304 "The internal helper macro HELPER_REGISTER_BINARY_FP_VP is already defined!"
305#endif
306#define HELPER_REGISTER_BINARY_FP_VP(OPSUFFIX, VPSD, IROPC, SDOPC) \
307 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 2, 3, VPSD, -1) \
308 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \
309 VP_PROPERTY_CONSTRAINEDFP(1, 1, experimental_constrained_##OPSUFFIX) \
310 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \
311 VP_PROPERTY_BINARYOP \
312 END_REGISTER_VP(vp_##OPSUFFIX, VPSD)
313
314// llvm.vp.fadd(x,y,mask,vlen)
315HELPER_REGISTER_BINARY_FP_VP(fadd, VP_FADD, FAdd, FADD)
316
317// llvm.vp.fsub(x,y,mask,vlen)
318HELPER_REGISTER_BINARY_FP_VP(fsub, VP_FSUB, FSub, FSUB)
319
320// llvm.vp.fmul(x,y,mask,vlen)
321HELPER_REGISTER_BINARY_FP_VP(fmul, VP_FMUL, FMul, FMUL)
322
323// llvm.vp.fdiv(x,y,mask,vlen)
324HELPER_REGISTER_BINARY_FP_VP(fdiv, VP_FDIV, FDiv, FDIV)
325
326// llvm.vp.frem(x,y,mask,vlen)
327HELPER_REGISTER_BINARY_FP_VP(frem, VP_FREM, FRem, FREM)
328
329#undef HELPER_REGISTER_BINARY_FP_VP
330
331// llvm.vp.fneg(x,mask,vlen)
332BEGIN_REGISTER_VP(vp_fneg, 1, 2, VP_FNEG, -1)
333VP_PROPERTY_FUNCTIONAL_OPC(FNeg)
334VP_PROPERTY_FUNCTIONAL_SDOPC(FNEG)
335END_REGISTER_VP(vp_fneg, VP_FNEG)
336
337// llvm.vp.fabs(x,mask,vlen)
338BEGIN_REGISTER_VP(vp_fabs, 1, 2, VP_FABS, -1)
339VP_PROPERTY_FUNCTIONAL_INTRINSIC(fabs)
340VP_PROPERTY_FUNCTIONAL_SDOPC(FABS)
341END_REGISTER_VP(vp_fabs, VP_FABS)
342
343// llvm.vp.sqrt(x,mask,vlen)
344BEGIN_REGISTER_VP(vp_sqrt, 1, 2, VP_SQRT, -1)
345VP_PROPERTY_FUNCTIONAL_INTRINSIC(sqrt)
346VP_PROPERTY_FUNCTIONAL_SDOPC(FSQRT)
347END_REGISTER_VP(vp_sqrt, VP_SQRT)
348
349// llvm.vp.fma(x,y,z,mask,vlen)
350BEGIN_REGISTER_VP(vp_fma, 3, 4, VP_FMA, -1)
351VP_PROPERTY_CONSTRAINEDFP(1, 1, experimental_constrained_fma)
352VP_PROPERTY_FUNCTIONAL_INTRINSIC(fma)
353VP_PROPERTY_FUNCTIONAL_SDOPC(FMA)
354END_REGISTER_VP(vp_fma, VP_FMA)
355
356// llvm.vp.fmuladd(x,y,z,mask,vlen)
357BEGIN_REGISTER_VP(vp_fmuladd, 3, 4, VP_FMULADD, -1)
358VP_PROPERTY_CONSTRAINEDFP(1, 1, experimental_constrained_fmuladd)
359VP_PROPERTY_FUNCTIONAL_INTRINSIC(fmuladd)
360VP_PROPERTY_FUNCTIONAL_SDOPC(FMAD)
361END_REGISTER_VP(vp_fmuladd, VP_FMULADD)
362
363// llvm.vp.copysign(x,y,mask,vlen)
364BEGIN_REGISTER_VP(vp_copysign, 2, 3, VP_FCOPYSIGN, -1)
365VP_PROPERTY_BINARYOP
366VP_PROPERTY_FUNCTIONAL_SDOPC(FCOPYSIGN)
367VP_PROPERTY_FUNCTIONAL_INTRINSIC(copysign)
368END_REGISTER_VP(vp_copysign, VP_FCOPYSIGN)
369
370// llvm.vp.minnum(x,y,mask,vlen)
371BEGIN_REGISTER_VP(vp_minnum, 2, 3, VP_FMINNUM, -1)
372VP_PROPERTY_BINARYOP
373VP_PROPERTY_FUNCTIONAL_SDOPC(FMINNUM)
374VP_PROPERTY_FUNCTIONAL_INTRINSIC(minnum)
375END_REGISTER_VP(vp_minnum, VP_FMINNUM)
376
377// llvm.vp.maxnum(x,y,mask,vlen)
378BEGIN_REGISTER_VP(vp_maxnum, 2, 3, VP_FMAXNUM, -1)
379VP_PROPERTY_BINARYOP
380VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXNUM)
381VP_PROPERTY_FUNCTIONAL_INTRINSIC(maxnum)
382END_REGISTER_VP(vp_maxnum, VP_FMAXNUM)
383
384// llvm.vp.minimum(x,y,mask,vlen)
385BEGIN_REGISTER_VP(vp_minimum, 2, 3, VP_FMINIMUM, -1)
386VP_PROPERTY_BINARYOP
387VP_PROPERTY_FUNCTIONAL_SDOPC(FMINIMUM)
388VP_PROPERTY_FUNCTIONAL_INTRINSIC(minimum)
389END_REGISTER_VP(vp_minimum, VP_FMINIMUM)
390
391// llvm.vp.maximum(x,y,mask,vlen)
392BEGIN_REGISTER_VP(vp_maximum, 2, 3, VP_FMAXIMUM, -1)
393VP_PROPERTY_BINARYOP
394VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXIMUM)
395VP_PROPERTY_FUNCTIONAL_INTRINSIC(maximum)
396END_REGISTER_VP(vp_maximum, VP_FMAXIMUM)
397
398// llvm.vp.ceil(x,mask,vlen)
399BEGIN_REGISTER_VP(vp_ceil, 1, 2, VP_FCEIL, -1)
400VP_PROPERTY_FUNCTIONAL_INTRINSIC(ceil)
401VP_PROPERTY_FUNCTIONAL_SDOPC(FCEIL)
402END_REGISTER_VP(vp_ceil, VP_FCEIL)
403
404// llvm.vp.floor(x,mask,vlen)
405BEGIN_REGISTER_VP(vp_floor, 1, 2, VP_FFLOOR, -1)
406VP_PROPERTY_FUNCTIONAL_INTRINSIC(floor)
407VP_PROPERTY_FUNCTIONAL_SDOPC(FFLOOR)
408END_REGISTER_VP(vp_floor, VP_FFLOOR)
409
410// llvm.vp.round(x,mask,vlen)
411BEGIN_REGISTER_VP(vp_round, 1, 2, VP_FROUND, -1)
412VP_PROPERTY_FUNCTIONAL_INTRINSIC(round)
413VP_PROPERTY_FUNCTIONAL_SDOPC(FROUND)
414END_REGISTER_VP(vp_round, VP_FROUND)
415
416// llvm.vp.roundeven(x,mask,vlen)
417BEGIN_REGISTER_VP(vp_roundeven, 1, 2, VP_FROUNDEVEN, -1)
418VP_PROPERTY_FUNCTIONAL_INTRINSIC(roundeven)
419VP_PROPERTY_FUNCTIONAL_SDOPC(FROUNDEVEN)
420END_REGISTER_VP(vp_roundeven, VP_FROUNDEVEN)
421
422// llvm.vp.roundtozero(x,mask,vlen)
423BEGIN_REGISTER_VP(vp_roundtozero, 1, 2, VP_FROUNDTOZERO, -1)
424VP_PROPERTY_FUNCTIONAL_INTRINSIC(trunc)
425VP_PROPERTY_FUNCTIONAL_SDOPC(FTRUNC)
426END_REGISTER_VP(vp_roundtozero, VP_FROUNDTOZERO)
427
428// llvm.vp.rint(x,mask,vlen)
429BEGIN_REGISTER_VP(vp_rint, 1, 2, VP_FRINT, -1)
430VP_PROPERTY_FUNCTIONAL_INTRINSIC(rint)
431VP_PROPERTY_FUNCTIONAL_SDOPC(FRINT)
432END_REGISTER_VP(vp_rint, VP_FRINT)
433
434// llvm.vp.nearbyint(x,mask,vlen)
435BEGIN_REGISTER_VP(vp_nearbyint, 1, 2, VP_FNEARBYINT, -1)
436VP_PROPERTY_FUNCTIONAL_INTRINSIC(nearbyint)
437VP_PROPERTY_FUNCTIONAL_SDOPC(FNEARBYINT)
438END_REGISTER_VP(vp_nearbyint, VP_FNEARBYINT)
439
440///// } Floating-Point Arithmetic
441
442///// Type Casts {
443// Specialized helper macro for type conversions.
444// <operation>(%x, %mask, %evl).
445#ifdef HELPER_REGISTER_FP_CAST_VP
446#error \
447 "The internal helper macro HELPER_REGISTER_FP_CAST_VP is already defined!"
448#endif
449#define HELPER_REGISTER_FP_CAST_VP(OPSUFFIX, VPSD, IROPC, SDOPC, HASROUND) \
450 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 1, 2, VPSD, -1) \
451 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \
452 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \
453 VP_PROPERTY_CONSTRAINEDFP(HASROUND, 1, experimental_constrained_##OPSUFFIX) \
454 VP_PROPERTY_CASTOP \
455 END_REGISTER_VP(vp_##OPSUFFIX, VPSD)
456
457// llvm.vp.fptoui(x,mask,vlen)
458HELPER_REGISTER_FP_CAST_VP(fptoui, VP_FP_TO_UINT, FPToUI, FP_TO_UINT, 0)
459
460// llvm.vp.fptosi(x,mask,vlen)
461HELPER_REGISTER_FP_CAST_VP(fptosi, VP_FP_TO_SINT, FPToSI, FP_TO_SINT, 0)
462
463// llvm.vp.uitofp(x,mask,vlen)
464HELPER_REGISTER_FP_CAST_VP(uitofp, VP_UINT_TO_FP, UIToFP, UINT_TO_FP, 1)
465
466// llvm.vp.sitofp(x,mask,vlen)
467HELPER_REGISTER_FP_CAST_VP(sitofp, VP_SINT_TO_FP, SIToFP, SINT_TO_FP, 1)
468
469// llvm.vp.fptrunc(x,mask,vlen)
470HELPER_REGISTER_FP_CAST_VP(fptrunc, VP_FP_ROUND, FPTrunc, FP_ROUND, 1)
471
472// llvm.vp.fpext(x,mask,vlen)
473HELPER_REGISTER_FP_CAST_VP(fpext, VP_FP_EXTEND, FPExt, FP_EXTEND, 0)
474
475#undef HELPER_REGISTER_FP_CAST_VP
476
477// Specialized helper macro for integer type conversions.
478// <operation>(%x, %mask, %evl).
479#ifdef HELPER_REGISTER_INT_CAST_VP
480#error \
481 "The internal helper macro HELPER_REGISTER_INT_CAST_VP is already defined!"
482#endif
483#define HELPER_REGISTER_INT_CAST_VP(OPSUFFIX, VPSD, IROPC, SDOPC) \
484 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 1, 2, VPSD, -1) \
485 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \
486 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \
487 VP_PROPERTY_CASTOP \
488 END_REGISTER_VP(vp_##OPSUFFIX, VPSD)
489
490// llvm.vp.trunc(x,mask,vlen)
491HELPER_REGISTER_INT_CAST_VP(trunc, VP_TRUNCATE, Trunc, TRUNCATE)
492
493// llvm.vp.zext(x,mask,vlen)
494HELPER_REGISTER_INT_CAST_VP(zext, VP_ZERO_EXTEND, ZExt, ZERO_EXTEND)
495
496// llvm.vp.sext(x,mask,vlen)
497HELPER_REGISTER_INT_CAST_VP(sext, VP_SIGN_EXTEND, SExt, SIGN_EXTEND)
498
499// llvm.vp.ptrtoint(x,mask,vlen)
500BEGIN_REGISTER_VP(vp_ptrtoint, 1, 2, VP_PTRTOINT, -1)
501VP_PROPERTY_FUNCTIONAL_OPC(PtrToInt)
502VP_PROPERTY_CASTOP
503END_REGISTER_VP(vp_ptrtoint, VP_PTRTOINT)
504
505// llvm.vp.inttoptr(x,mask,vlen)
506BEGIN_REGISTER_VP(vp_inttoptr, 1, 2, VP_INTTOPTR, -1)
507VP_PROPERTY_FUNCTIONAL_OPC(IntToPtr)
508VP_PROPERTY_CASTOP
509END_REGISTER_VP(vp_inttoptr, VP_INTTOPTR)
510
511#undef HELPER_REGISTER_INT_CAST_VP
512
513///// } Type Casts
514
515///// Comparisons {
516
517// VP_SETCC (ISel only)
518BEGIN_REGISTER_VP_SDNODE(VP_SETCC, 0, vp_setcc, 3, 4)
519END_REGISTER_VP_SDNODE(VP_SETCC)
520
521// llvm.vp.fcmp(x,y,cc,mask,vlen)
522BEGIN_REGISTER_VP_INTRINSIC(vp_fcmp, 3, 4)
523HELPER_MAP_VPID_TO_VPSD(vp_fcmp, VP_SETCC)
524VP_PROPERTY_FUNCTIONAL_OPC(FCmp)
525VP_PROPERTY_CMP(2, true)
526VP_PROPERTY_CONSTRAINEDFP(0, 1, experimental_constrained_fcmp)
527END_REGISTER_VP_INTRINSIC(vp_fcmp)
528
529// llvm.vp.icmp(x,y,cc,mask,vlen)
530BEGIN_REGISTER_VP_INTRINSIC(vp_icmp, 3, 4)
531HELPER_MAP_VPID_TO_VPSD(vp_icmp, VP_SETCC)
532VP_PROPERTY_FUNCTIONAL_OPC(ICmp)
533VP_PROPERTY_CMP(2, false)
534END_REGISTER_VP_INTRINSIC(vp_icmp)
535
536///// } Comparisons
537
538// llvm.vp.is.fpclass(on_true,on_false,mask,vlen)
539BEGIN_REGISTER_VP(vp_is_fpclass, 2, 3, VP_IS_FPCLASS, 0)
540VP_PROPERTY_FUNCTIONAL_INTRINSIC(is_fpclass)
541END_REGISTER_VP(vp_is_fpclass, VP_IS_FPCLASS)
542
543///// Memory Operations {
544// llvm.vp.store(val,ptr,mask,vlen)
545BEGIN_REGISTER_VP_INTRINSIC(vp_store, 2, 3)
546// chain = VP_STORE chain,val,base,offset,mask,evl
547BEGIN_REGISTER_VP_SDNODE(VP_STORE, 1, vp_store, 4, 5)
548HELPER_MAP_VPID_TO_VPSD(vp_store, VP_STORE)
549VP_PROPERTY_FUNCTIONAL_OPC(Store)
550VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_store)
551VP_PROPERTY_MEMOP(1, 0)
552END_REGISTER_VP(vp_store, VP_STORE)
553
554// llvm.experimental.vp.strided.store(val,ptr,stride,mask,vlen)
555BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_strided_store, 3, 4)
556// chain = EXPERIMENTAL_VP_STRIDED_STORE chain,val,base,offset,stride,mask,evl
557VP_PROPERTY_NO_FUNCTIONAL
558BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_STRIDED_STORE, 1, experimental_vp_strided_store, 5, 6)
559HELPER_MAP_VPID_TO_VPSD(experimental_vp_strided_store, EXPERIMENTAL_VP_STRIDED_STORE)
560VP_PROPERTY_MEMOP(1, 0)
561END_REGISTER_VP(experimental_vp_strided_store, EXPERIMENTAL_VP_STRIDED_STORE)
562
563// llvm.vp.scatter(ptr,val,mask,vlen)
564BEGIN_REGISTER_VP_INTRINSIC(vp_scatter, 2, 3)
565// chain = VP_SCATTER chain,val,base,indices,scale,mask,evl
566BEGIN_REGISTER_VP_SDNODE(VP_SCATTER, 1, vp_scatter, 5, 6)
567HELPER_MAP_VPID_TO_VPSD(vp_scatter, VP_SCATTER)
568VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_scatter)
569VP_PROPERTY_MEMOP(1, 0)
570END_REGISTER_VP(vp_scatter, VP_SCATTER)
571
572// llvm.vp.load(ptr,mask,vlen)
573BEGIN_REGISTER_VP_INTRINSIC(vp_load, 1, 2)
574// val,chain = VP_LOAD chain,base,offset,mask,evl
575BEGIN_REGISTER_VP_SDNODE(VP_LOAD, -1, vp_load, 3, 4)
576HELPER_MAP_VPID_TO_VPSD(vp_load, VP_LOAD)
577VP_PROPERTY_FUNCTIONAL_OPC(Load)
578VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_load)
579VP_PROPERTY_MEMOP(0, std::nullopt)
580END_REGISTER_VP(vp_load, VP_LOAD)
581
582// llvm.experimental.vp.strided.load(ptr,stride,mask,vlen)
583BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_strided_load, 2, 3)
584// chain = EXPERIMENTAL_VP_STRIDED_LOAD chain,base,offset,stride,mask,evl
585VP_PROPERTY_NO_FUNCTIONAL
586BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_STRIDED_LOAD, -1, experimental_vp_strided_load, 4, 5)
587HELPER_MAP_VPID_TO_VPSD(experimental_vp_strided_load, EXPERIMENTAL_VP_STRIDED_LOAD)
588VP_PROPERTY_MEMOP(0, std::nullopt)
589END_REGISTER_VP(experimental_vp_strided_load, EXPERIMENTAL_VP_STRIDED_LOAD)
590
591// llvm.vp.gather(ptr,mask,vlen)
592BEGIN_REGISTER_VP_INTRINSIC(vp_gather, 1, 2)
593// val,chain = VP_GATHER chain,base,indices,scale,mask,evl
594BEGIN_REGISTER_VP_SDNODE(VP_GATHER, -1, vp_gather, 4, 5)
595HELPER_MAP_VPID_TO_VPSD(vp_gather, VP_GATHER)
596VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_gather)
597VP_PROPERTY_MEMOP(0, std::nullopt)
598END_REGISTER_VP(vp_gather, VP_GATHER)
599
600///// } Memory Operations
601
602///// Reductions {
603
604// Specialized helper macro for VP reductions (%start, %x, %mask, %evl).
605#ifdef HELPER_REGISTER_REDUCTION_VP
606#error \
607 "The internal helper macro HELPER_REGISTER_REDUCTION_VP is already defined!"
608#endif
609#define HELPER_REGISTER_REDUCTION_VP(VPID, VPSD, INTRIN) \
610 BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, 1) \
611 VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \
612 VP_PROPERTY_REDUCTION(0, 1) \
613 END_REGISTER_VP(VPID, VPSD)
614
615// llvm.vp.reduce.add(start,x,mask,vlen)
616HELPER_REGISTER_REDUCTION_VP(vp_reduce_add, VP_REDUCE_ADD,
617 vector_reduce_add)
618
619// llvm.vp.reduce.mul(start,x,mask,vlen)
620HELPER_REGISTER_REDUCTION_VP(vp_reduce_mul, VP_REDUCE_MUL,
621 vector_reduce_mul)
622
623// llvm.vp.reduce.and(start,x,mask,vlen)
624HELPER_REGISTER_REDUCTION_VP(vp_reduce_and, VP_REDUCE_AND,
625 vector_reduce_and)
626
627// llvm.vp.reduce.or(start,x,mask,vlen)
628HELPER_REGISTER_REDUCTION_VP(vp_reduce_or, VP_REDUCE_OR,
629 vector_reduce_or)
630
631// llvm.vp.reduce.xor(start,x,mask,vlen)
632HELPER_REGISTER_REDUCTION_VP(vp_reduce_xor, VP_REDUCE_XOR,
633 vector_reduce_xor)
634
635// llvm.vp.reduce.smax(start,x,mask,vlen)
636HELPER_REGISTER_REDUCTION_VP(vp_reduce_smax, VP_REDUCE_SMAX,
637 vector_reduce_smax)
638
639// llvm.vp.reduce.smin(start,x,mask,vlen)
640HELPER_REGISTER_REDUCTION_VP(vp_reduce_smin, VP_REDUCE_SMIN,
641 vector_reduce_smin)
642
643// llvm.vp.reduce.umax(start,x,mask,vlen)
644HELPER_REGISTER_REDUCTION_VP(vp_reduce_umax, VP_REDUCE_UMAX,
645 vector_reduce_umax)
646
647// llvm.vp.reduce.umin(start,x,mask,vlen)
648HELPER_REGISTER_REDUCTION_VP(vp_reduce_umin, VP_REDUCE_UMIN,
649 vector_reduce_umin)
650
651// llvm.vp.reduce.fmax(start,x,mask,vlen)
652HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmax, VP_REDUCE_FMAX,
653 vector_reduce_fmax)
654
655// llvm.vp.reduce.fmin(start,x,mask,vlen)
656HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmin, VP_REDUCE_FMIN,
657 vector_reduce_fmin)
658
659#undef HELPER_REGISTER_REDUCTION_VP
660
661// Specialized helper macro for VP reductions as above but with two forms:
662// sequential and reassociative. These manifest as the presence of 'reassoc'
663// fast-math flags in the IR and as two distinct ISD opcodes in the
664// SelectionDAG.
665// Note we by default map from the VP intrinsic to the SEQ ISD opcode, which
666// can then be relaxed to the non-SEQ ISD opcode if the 'reassoc' flag is set.
667#ifdef HELPER_REGISTER_REDUCTION_SEQ_VP
668#error \
669 "The internal helper macro HELPER_REGISTER_REDUCTION_SEQ_VP is already defined!"
670#endif
671#define HELPER_REGISTER_REDUCTION_SEQ_VP(VPID, VPSD, SEQ_VPSD, INTRIN) \
672 BEGIN_REGISTER_VP_INTRINSIC(VPID, 2, 3) \
673 BEGIN_REGISTER_VP_SDNODE(VPSD, 1, VPID, 2, 3) \
674 VP_PROPERTY_REDUCTION(0, 1) \
675 END_REGISTER_VP_SDNODE(VPSD) \
676 BEGIN_REGISTER_VP_SDNODE(SEQ_VPSD, 1, VPID, 2, 3) \
677 HELPER_MAP_VPID_TO_VPSD(VPID, SEQ_VPSD) \
678 VP_PROPERTY_REDUCTION(0, 1) \
679 END_REGISTER_VP_SDNODE(SEQ_VPSD) \
680 VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \
681 END_REGISTER_VP_INTRINSIC(VPID)
682
683// llvm.vp.reduce.fadd(start,x,mask,vlen)
684HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fadd, VP_REDUCE_FADD,
685 VP_REDUCE_SEQ_FADD,
686 vector_reduce_fadd)
687
688// llvm.vp.reduce.fmul(start,x,mask,vlen)
689HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fmul, VP_REDUCE_FMUL,
690 VP_REDUCE_SEQ_FMUL,
691 vector_reduce_fmul)
692
693#undef HELPER_REGISTER_REDUCTION_SEQ_VP
694
695///// } Reduction
696
697///// Shuffles {
698
699// The mask 'cond' operand of llvm.vp.select and llvm.vp.merge are not reported
700// as masks with the BEGIN_REGISTER_VP_* macros. This is because, unlike other
701// VP intrinsics, these two have a defined result on lanes where the mask is
702// false.
703//
704// llvm.vp.select(cond,on_true,on_false,vlen)
705BEGIN_REGISTER_VP(vp_select, std::nullopt, 3, VP_SELECT, -1)
706VP_PROPERTY_FUNCTIONAL_OPC(Select)
707VP_PROPERTY_FUNCTIONAL_SDOPC(VSELECT)
708END_REGISTER_VP(vp_select, VP_SELECT)
709
710// llvm.vp.merge(cond,on_true,on_false,pivot)
711BEGIN_REGISTER_VP(vp_merge, std::nullopt, 3, VP_MERGE, -1)
712VP_PROPERTY_NO_FUNCTIONAL
713END_REGISTER_VP(vp_merge, VP_MERGE)
714
715BEGIN_REGISTER_VP(experimental_vp_splice, 3, 5, EXPERIMENTAL_VP_SPLICE, -1)
716VP_PROPERTY_NO_FUNCTIONAL
717END_REGISTER_VP(experimental_vp_splice, EXPERIMENTAL_VP_SPLICE)
718
719// llvm.experimental.vp.reverse(x,mask,vlen)
720BEGIN_REGISTER_VP(experimental_vp_reverse, 1, 2,
721 EXPERIMENTAL_VP_REVERSE, -1)
722VP_PROPERTY_NO_FUNCTIONAL
723END_REGISTER_VP(experimental_vp_reverse, EXPERIMENTAL_VP_REVERSE)
724
725///// } Shuffles
726
727#undef BEGIN_REGISTER_VP
728#undef BEGIN_REGISTER_VP_INTRINSIC
729#undef BEGIN_REGISTER_VP_SDNODE
730#undef END_REGISTER_VP
731#undef END_REGISTER_VP_INTRINSIC
732#undef END_REGISTER_VP_SDNODE
733#undef HELPER_MAP_VPID_TO_VPSD
734#undef VP_PROPERTY_BINARYOP
735#undef VP_PROPERTY_CASTOP
736#undef VP_PROPERTY_CMP
737#undef VP_PROPERTY_CONSTRAINEDFP
738#undef VP_PROPERTY_FUNCTIONAL_INTRINSIC
739#undef VP_PROPERTY_FUNCTIONAL_OPC
740#undef VP_PROPERTY_FUNCTIONAL_SDOPC
741#undef VP_PROPERTY_NO_FUNCTIONAL
742#undef VP_PROPERTY_MEMOP
743#undef VP_PROPERTY_REDUCTION
744

source code of llvm/include/llvm/IR/VPIntrinsics.def