1/* Schedule GIMPLE vector statements.
2 Copyright (C) 2020-2023 Free Software Foundation, Inc.
3
4This file is part of GCC.
5
6GCC is free software; you can redistribute it and/or modify it
7under the terms of the GNU General Public License as published by the
8Free Software Foundation; either version 3, or (at your option) any
9later version.
10
11GCC is distributed in the hope that it will be useful, but WITHOUT
12ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14for more details.
15
16You should have received a copy of the GNU General Public License
17along with GCC; see the file COPYING3. If not see
18<http://www.gnu.org/licenses/>. */
19
20#include "config.h"
21#include "system.h"
22#include "coretypes.h"
23#include "backend.h"
24#include "rtl.h"
25#include "tree.h"
26#include "gimple.h"
27#include "tree-pass.h"
28#include "ssa.h"
29#include "expmed.h"
30#include "optabs-tree.h"
31#include "tree-eh.h"
32#include "gimple-iterator.h"
33#include "gimplify-me.h"
34#include "gimplify.h"
35#include "tree-cfg.h"
36#include "bitmap.h"
37#include "tree-ssa-dce.h"
38#include "memmodel.h"
39#include "optabs.h"
40#include "gimple-fold.h"
41#include "internal-fn.h"
42
43/* Expand all ARRAY_REF(VIEW_CONVERT_EXPR) gimple assignments into calls to
44 internal function based on vector type of selected expansion.
45
46 For vec_set:
47
48 VIEW_CONVERT_EXPR<int[4]>(u)[_1] = i_4(D);
49 =>
50 _7 = u;
51 _8 = .VEC_SET (_7, i_4(D), _1);
52 u = _8;
53
54 For vec_extract:
55
56 _3 = VIEW_CONVERT_EXPR<intD.1[4]>(vD.2208)[idx_2(D)];
57 =>
58 _4 = vD.2208;
59 _3 = .VEC_EXTRACT (_4, idx_2(D)); */
60
61static bool
62gimple_expand_vec_set_extract_expr (struct function *fun,
63 gimple_stmt_iterator *gsi)
64{
65 gcall *new_stmt = NULL;
66 gassign *ass_stmt = NULL;
67 bool cfg_changed = false;
68
69 /* Only consider code == GIMPLE_ASSIGN. */
70 gassign *stmt = dyn_cast<gassign *> (p: gsi_stmt (i: *gsi));
71 if (!stmt)
72 return false;
73
74 bool is_extract = false;
75
76 tree lhs = gimple_assign_lhs (gs: stmt);
77 tree rhs = gimple_assign_rhs1 (gs: stmt);
78 tree val, ref;
79 if (TREE_CODE (lhs) == ARRAY_REF)
80 {
81 /* Assume it is a vec_set. */
82 val = rhs;
83 ref = lhs;
84 }
85 else if (TREE_CODE (rhs) == ARRAY_REF)
86 {
87 /* vec_extract. */
88 is_extract = true;
89 val = lhs;
90 ref = rhs;
91 }
92 else
93 return false;
94
95 tree op0 = TREE_OPERAND (ref, 0);
96 if (TREE_CODE (op0) == VIEW_CONVERT_EXPR && DECL_P (TREE_OPERAND (op0, 0))
97 && VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (op0, 0)))
98 && TYPE_MODE (TREE_TYPE (ref))
99 == TYPE_MODE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (op0, 0)))))
100 {
101 tree pos = TREE_OPERAND (ref, 1);
102
103 tree view_op0 = TREE_OPERAND (op0, 0);
104 machine_mode outermode = TYPE_MODE (TREE_TYPE (view_op0));
105 machine_mode extract_mode = TYPE_MODE (TREE_TYPE (ref));
106
107 if (auto_var_in_fn_p (view_op0, fun->decl)
108 && !TREE_ADDRESSABLE (view_op0)
109 && ((!is_extract && can_vec_set_var_idx_p (outermode))
110 || (is_extract
111 && can_vec_extract_var_idx_p (outermode, extract_mode))))
112 {
113 location_t loc = gimple_location (g: stmt);
114 tree var_src = make_ssa_name (TREE_TYPE (view_op0));
115
116 ass_stmt = gimple_build_assign (var_src, view_op0);
117 gimple_set_vuse (g: ass_stmt, vuse: gimple_vuse (g: stmt));
118 gimple_set_location (g: ass_stmt, location: loc);
119 gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
120
121 if (!is_extract)
122 {
123 tree var_dst = make_ssa_name (TREE_TYPE (view_op0));
124
125 new_stmt = gimple_build_call_internal (IFN_VEC_SET, 3, var_src,
126 val, pos);
127
128 gimple_call_set_lhs (gs: new_stmt, lhs: var_dst);
129 gimple_set_location (g: new_stmt, location: loc);
130 gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
131
132 ass_stmt = gimple_build_assign (view_op0, var_dst);
133 gimple_set_location (g: ass_stmt, location: loc);
134 gimple_move_vops (ass_stmt, stmt);
135 gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
136
137 basic_block bb = gimple_bb (g: stmt);
138 if (gsi_remove (gsi, true)
139 && gimple_purge_dead_eh_edges (bb))
140 cfg_changed = true;
141 *gsi = gsi_for_stmt (ass_stmt);
142 }
143 else
144 {
145 new_stmt
146 = gimple_build_call_internal (IFN_VEC_EXTRACT, 2, var_src, pos);
147 gimple_call_set_lhs (gs: new_stmt, lhs);
148
149 gsi_replace (gsi, new_stmt, true);
150 cfg_changed = true;
151 }
152 }
153 }
154
155 return cfg_changed;
156}
157
158/* Expand all VEC_COND_EXPR gimple assignments into calls to internal
159 function based on type of selected expansion. */
160
161static gimple *
162gimple_expand_vec_cond_expr (struct function *fun, gimple_stmt_iterator *gsi,
163 hash_map<tree, unsigned int> *vec_cond_ssa_name_uses)
164{
165 tree lhs, op0a = NULL_TREE, op0b = NULL_TREE;
166 enum tree_code code;
167 enum tree_code tcode;
168 machine_mode cmp_op_mode;
169 bool unsignedp;
170 enum insn_code icode;
171 imm_use_iterator imm_iter;
172
173 /* Only consider code == GIMPLE_ASSIGN. */
174 gassign *stmt = dyn_cast<gassign *> (p: gsi_stmt (i: *gsi));
175 if (!stmt)
176 return NULL;
177
178 code = gimple_assign_rhs_code (gs: stmt);
179 if (code != VEC_COND_EXPR)
180 return NULL;
181
182 tree op0 = gimple_assign_rhs1 (gs: stmt);
183 tree op1 = gimple_assign_rhs2 (gs: stmt);
184 tree op2 = gimple_assign_rhs3 (gs: stmt);
185 lhs = gimple_assign_lhs (gs: stmt);
186 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
187
188 /* Lower mask typed, non-vector mode VEC_COND_EXPRs to bitwise operations.
189 Those can end up generated by folding and at least for integer mode masks
190 we cannot expect vcond expanders to exist. We lower a ? b : c
191 to (b & a) | (c & ~a). */
192 if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (lhs))
193 && !VECTOR_MODE_P (mode))
194 {
195 gcc_assert (types_compatible_p (TREE_TYPE (op0), TREE_TYPE (op1)));
196 gimple_seq stmts = NULL;
197 tree type = TREE_TYPE (lhs);
198 location_t loc = gimple_location (g: stmt);
199 tree tem0 = gimple_build (seq: &stmts, loc, code: BIT_AND_EXPR, type, ops: op1, ops: op0);
200 tree tem1 = gimple_build (seq: &stmts, loc, code: BIT_NOT_EXPR, type, ops: op0);
201 tree tem2 = gimple_build (seq: &stmts, loc, code: BIT_AND_EXPR, type, ops: op2, ops: tem1);
202 tree tem3 = gimple_build (seq: &stmts, loc, code: BIT_IOR_EXPR, type, ops: tem0, ops: tem2);
203 gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
204 return gimple_build_assign (lhs, tem3);
205 }
206
207 bool can_compute_op0 = true;
208 gcc_assert (!COMPARISON_CLASS_P (op0));
209 if (TREE_CODE (op0) == SSA_NAME)
210 {
211 unsigned int used_vec_cond_exprs = 0;
212 unsigned int *slot = vec_cond_ssa_name_uses->get (k: op0);
213 if (slot)
214 used_vec_cond_exprs = *slot;
215 else
216 {
217 gimple *use_stmt;
218 FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, op0)
219 {
220 gassign *assign = dyn_cast<gassign *> (p: use_stmt);
221 if (assign != NULL
222 && gimple_assign_rhs_code (gs: assign) == VEC_COND_EXPR
223 && gimple_assign_rhs1 (gs: assign) == op0)
224 used_vec_cond_exprs++;
225 }
226 vec_cond_ssa_name_uses->put (k: op0, v: used_vec_cond_exprs);
227 }
228
229 gassign *def_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op0));
230 if (def_stmt)
231 {
232 tcode = gimple_assign_rhs_code (gs: def_stmt);
233 op0a = gimple_assign_rhs1 (gs: def_stmt);
234 op0b = gimple_assign_rhs2 (gs: def_stmt);
235
236 tree op0_type = TREE_TYPE (op0);
237 tree op0a_type = TREE_TYPE (op0a);
238 if (TREE_CODE_CLASS (tcode) == tcc_comparison)
239 can_compute_op0 = expand_vec_cmp_expr_p (op0a_type, op0_type,
240 tcode);
241
242 /* Try to fold x CMP y ? -1 : 0 to x CMP y. */
243 if (can_compute_op0
244 && integer_minus_onep (op1)
245 && integer_zerop (op2)
246 && TYPE_MODE (TREE_TYPE (lhs)) == TYPE_MODE (TREE_TYPE (op0)))
247 {
248 tree conv_op = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), op0);
249 gassign *new_stmt = gimple_build_assign (lhs, conv_op);
250 gsi_replace (gsi, new_stmt, true);
251 return new_stmt;
252 }
253
254 /* When the compare has EH we do not want to forward it when
255 it has multiple uses and in general because of the complication
256 with EH redirection. */
257 if (stmt_can_throw_internal (fun, def_stmt))
258 tcode = TREE_CODE (op0);
259
260 /* If we can compute op0 and have multiple uses, keep the SSA
261 name and use vcond_mask. */
262 else if (can_compute_op0
263 && used_vec_cond_exprs >= 2
264 && (get_vcond_mask_icode (vmode: mode, TYPE_MODE (op0_type))
265 != CODE_FOR_nothing))
266 tcode = TREE_CODE (op0);
267 }
268 else
269 tcode = TREE_CODE (op0);
270 }
271 else
272 tcode = TREE_CODE (op0);
273
274 if (TREE_CODE_CLASS (tcode) != tcc_comparison)
275 {
276 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
277 if (get_vcond_mask_icode (vmode: mode, TYPE_MODE (TREE_TYPE (op0)))
278 != CODE_FOR_nothing)
279 return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
280 /* Fake op0 < 0. */
281 else
282 {
283 gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
284 == MODE_VECTOR_INT);
285 op0a = op0;
286 op0b = build_zero_cst (TREE_TYPE (op0));
287 tcode = LT_EXPR;
288 }
289 }
290 cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
291 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
292
293 gcc_assert (known_eq (GET_MODE_NUNITS (mode),
294 GET_MODE_NUNITS (cmp_op_mode)));
295
296 icode = get_vcond_icode (vmode: mode, cmode: cmp_op_mode, uns: unsignedp);
297 /* Some targets do not have vcondeq and only vcond with NE/EQ
298 but not vcondu, so make sure to also try vcond here as
299 vcond_icode_p would canonicalize the optab query to. */
300 if (icode == CODE_FOR_nothing
301 && (tcode == NE_EXPR || tcode == EQ_EXPR)
302 && ((icode = get_vcond_icode (vmode: mode, cmode: cmp_op_mode, uns: !unsignedp))
303 != CODE_FOR_nothing))
304 unsignedp = !unsignedp;
305 if (icode == CODE_FOR_nothing)
306 {
307 if (tcode == LT_EXPR
308 && op0a == op0)
309 {
310 /* A VEC_COND_EXPR condition could be folded from EQ_EXPR/NE_EXPR
311 into a constant when only get_vcond_eq_icode is supported.
312 Try changing it to NE_EXPR. */
313 tcode = NE_EXPR;
314 }
315 if ((tcode == EQ_EXPR || tcode == NE_EXPR)
316 && direct_internal_fn_supported_p (fn: IFN_VCONDEQ, TREE_TYPE (lhs),
317 TREE_TYPE (op0a),
318 opt_type: OPTIMIZE_FOR_BOTH))
319 {
320 tree tcode_tree = build_int_cst (integer_type_node, tcode);
321 return gimple_build_call_internal (IFN_VCONDEQ, 5, op0a, op0b, op1,
322 op2, tcode_tree);
323 }
324
325 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))
326 && can_compute_op0
327 && (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
328 != CODE_FOR_nothing));
329 return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
330 }
331
332 tree tcode_tree = build_int_cst (integer_type_node, tcode);
333 return gimple_build_call_internal (unsignedp ? IFN_VCONDU : IFN_VCOND,
334 5, op0a, op0b, op1, op2, tcode_tree);
335}
336
337
338
339namespace {
340
341const pass_data pass_data_gimple_isel =
342{
343 .type: GIMPLE_PASS, /* type */
344 .name: "isel", /* name */
345 .optinfo_flags: OPTGROUP_VEC, /* optinfo_flags */
346 .tv_id: TV_NONE, /* tv_id */
347 PROP_cfg, /* properties_required */
348 .properties_provided: 0, /* properties_provided */
349 .properties_destroyed: 0, /* properties_destroyed */
350 .todo_flags_start: 0, /* todo_flags_start */
351 TODO_update_ssa, /* todo_flags_finish */
352};
353
354class pass_gimple_isel : public gimple_opt_pass
355{
356public:
357 pass_gimple_isel (gcc::context *ctxt)
358 : gimple_opt_pass (pass_data_gimple_isel, ctxt)
359 {}
360
361 /* opt_pass methods: */
362 bool gate (function *) final override
363 {
364 return true;
365 }
366
367 unsigned int execute (function *fun) final override;
368}; // class pass_gimple_isel
369
370
371/* Iterate all gimple statements and perform pre RTL expansion
372 GIMPLE massaging to improve instruction selection. */
373
374unsigned int
375pass_gimple_isel::execute (struct function *fun)
376{
377 gimple_stmt_iterator gsi;
378 basic_block bb;
379 hash_map<tree, unsigned int> vec_cond_ssa_name_uses;
380 auto_bitmap dce_ssa_names;
381 bool cfg_changed = false;
382
383 FOR_EACH_BB_FN (bb, fun)
384 {
385 for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
386 {
387 /* Pre-expand VEC_COND_EXPRs to .VCOND* internal function
388 calls mapping to supported optabs. */
389 gimple *g = gimple_expand_vec_cond_expr (fun, gsi: &gsi,
390 vec_cond_ssa_name_uses: &vec_cond_ssa_name_uses);
391 if (g != NULL)
392 {
393 tree lhs = gimple_assign_lhs (gs: gsi_stmt (i: gsi));
394 gimple_set_lhs (g, lhs);
395 gsi_replace (&gsi, g, false);
396 }
397
398 /* Recognize .VEC_SET and .VEC_EXTRACT patterns. */
399 cfg_changed |= gimple_expand_vec_set_extract_expr (fun, gsi: &gsi);
400 if (gsi_end_p (i: gsi))
401 break;
402
403 gassign *stmt = dyn_cast <gassign *> (p: *gsi);
404 if (!stmt)
405 continue;
406
407 tree_code code = gimple_assign_rhs_code (gs: stmt);
408 tree lhs = gimple_assign_lhs (gs: stmt);
409 if (TREE_CODE_CLASS (code) == tcc_comparison
410 && !has_single_use (var: lhs))
411 {
412 /* Duplicate COND_EXPR condition defs when they are
413 comparisons so RTL expansion with the help of TER
414 can perform better if conversion. */
415 imm_use_iterator imm_iter;
416 use_operand_p use_p;
417 auto_vec<gassign *, 4> cond_exprs;
418 unsigned cnt = 0;
419 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
420 {
421 if (is_gimple_debug (USE_STMT (use_p)))
422 continue;
423 cnt++;
424 if (gimple_bb (USE_STMT (use_p)) == bb
425 && is_gimple_assign (USE_STMT (use_p))
426 && gimple_assign_rhs1_ptr (USE_STMT (use_p)) == use_p->use
427 && gimple_assign_rhs_code (USE_STMT (use_p)) == COND_EXPR)
428 cond_exprs.safe_push (obj: as_a <gassign *> (USE_STMT (use_p)));
429 }
430 for (unsigned i = cond_exprs.length () == cnt ? 1 : 0;
431 i < cond_exprs.length (); ++i)
432 {
433 gassign *copy = as_a <gassign *> (p: gimple_copy (stmt));
434 tree new_def = duplicate_ssa_name (var: lhs, stmt: copy);
435 gimple_assign_set_lhs (gs: copy, lhs: new_def);
436 auto gsi2 = gsi_for_stmt (cond_exprs[i]);
437 gsi_insert_before (&gsi2, copy, GSI_SAME_STMT);
438 gimple_assign_set_rhs1 (gs: cond_exprs[i], rhs: new_def);
439 update_stmt (s: cond_exprs[i]);
440 }
441 }
442 }
443 }
444
445 for (auto it = vec_cond_ssa_name_uses.begin ();
446 it != vec_cond_ssa_name_uses.end (); ++it)
447 bitmap_set_bit (dce_ssa_names, SSA_NAME_VERSION ((*it).first));
448
449 simple_dce_from_worklist (dce_ssa_names);
450
451 return cfg_changed ? TODO_cleanup_cfg : 0;
452}
453
454} // anon namespace
455
456gimple_opt_pass *
457make_pass_gimple_isel (gcc::context *ctxt)
458{
459 return new pass_gimple_isel (ctxt);
460}
461
462

source code of gcc/gimple-isel.cc