1 | /* Schedule GIMPLE vector statements. |
2 | Copyright (C) 2020-2023 Free Software Foundation, Inc. |
3 | |
4 | This file is part of GCC. |
5 | |
6 | GCC is free software; you can redistribute it and/or modify it |
7 | under the terms of the GNU General Public License as published by the |
8 | Free Software Foundation; either version 3, or (at your option) any |
9 | later version. |
10 | |
11 | GCC is distributed in the hope that it will be useful, but WITHOUT |
12 | ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
13 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
14 | for more details. |
15 | |
16 | You should have received a copy of the GNU General Public License |
17 | along with GCC; see the file COPYING3. If not see |
18 | <http://www.gnu.org/licenses/>. */ |
19 | |
20 | #include "config.h" |
21 | #include "system.h" |
22 | #include "coretypes.h" |
23 | #include "backend.h" |
24 | #include "rtl.h" |
25 | #include "tree.h" |
26 | #include "gimple.h" |
27 | #include "tree-pass.h" |
28 | #include "ssa.h" |
29 | #include "expmed.h" |
30 | #include "optabs-tree.h" |
31 | #include "tree-eh.h" |
32 | #include "gimple-iterator.h" |
33 | #include "gimplify-me.h" |
34 | #include "gimplify.h" |
35 | #include "tree-cfg.h" |
36 | #include "bitmap.h" |
37 | #include "tree-ssa-dce.h" |
38 | #include "memmodel.h" |
39 | #include "optabs.h" |
40 | #include "gimple-fold.h" |
41 | #include "internal-fn.h" |
42 | |
43 | /* Expand all ARRAY_REF(VIEW_CONVERT_EXPR) gimple assignments into calls to |
44 | internal function based on vector type of selected expansion. |
45 | |
46 | For vec_set: |
47 | |
48 | VIEW_CONVERT_EXPR<int[4]>(u)[_1] = i_4(D); |
49 | => |
50 | _7 = u; |
51 | _8 = .VEC_SET (_7, i_4(D), _1); |
52 | u = _8; |
53 | |
54 | For vec_extract: |
55 | |
56 | _3 = VIEW_CONVERT_EXPR<intD.1[4]>(vD.2208)[idx_2(D)]; |
57 | => |
58 | _4 = vD.2208; |
59 | _3 = .VEC_EXTRACT (_4, idx_2(D)); */ |
60 | |
61 | static bool |
62 | gimple_expand_vec_set_extract_expr (struct function *fun, |
63 | gimple_stmt_iterator *gsi) |
64 | { |
65 | gcall *new_stmt = NULL; |
66 | gassign *ass_stmt = NULL; |
67 | bool cfg_changed = false; |
68 | |
69 | /* Only consider code == GIMPLE_ASSIGN. */ |
70 | gassign *stmt = dyn_cast<gassign *> (p: gsi_stmt (i: *gsi)); |
71 | if (!stmt) |
72 | return false; |
73 | |
74 | bool = false; |
75 | |
76 | tree lhs = gimple_assign_lhs (gs: stmt); |
77 | tree rhs = gimple_assign_rhs1 (gs: stmt); |
78 | tree val, ref; |
79 | if (TREE_CODE (lhs) == ARRAY_REF) |
80 | { |
81 | /* Assume it is a vec_set. */ |
82 | val = rhs; |
83 | ref = lhs; |
84 | } |
85 | else if (TREE_CODE (rhs) == ARRAY_REF) |
86 | { |
87 | /* vec_extract. */ |
88 | is_extract = true; |
89 | val = lhs; |
90 | ref = rhs; |
91 | } |
92 | else |
93 | return false; |
94 | |
95 | tree op0 = TREE_OPERAND (ref, 0); |
96 | if (TREE_CODE (op0) == VIEW_CONVERT_EXPR && DECL_P (TREE_OPERAND (op0, 0)) |
97 | && VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (op0, 0))) |
98 | && TYPE_MODE (TREE_TYPE (ref)) |
99 | == TYPE_MODE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (op0, 0))))) |
100 | { |
101 | tree pos = TREE_OPERAND (ref, 1); |
102 | |
103 | tree view_op0 = TREE_OPERAND (op0, 0); |
104 | machine_mode outermode = TYPE_MODE (TREE_TYPE (view_op0)); |
105 | machine_mode = TYPE_MODE (TREE_TYPE (ref)); |
106 | |
107 | if (auto_var_in_fn_p (view_op0, fun->decl) |
108 | && !TREE_ADDRESSABLE (view_op0) |
109 | && ((!is_extract && can_vec_set_var_idx_p (outermode)) |
110 | || (is_extract |
111 | && can_vec_extract_var_idx_p (outermode, extract_mode)))) |
112 | { |
113 | location_t loc = gimple_location (g: stmt); |
114 | tree var_src = make_ssa_name (TREE_TYPE (view_op0)); |
115 | |
116 | ass_stmt = gimple_build_assign (var_src, view_op0); |
117 | gimple_set_vuse (g: ass_stmt, vuse: gimple_vuse (g: stmt)); |
118 | gimple_set_location (g: ass_stmt, location: loc); |
119 | gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT); |
120 | |
121 | if (!is_extract) |
122 | { |
123 | tree var_dst = make_ssa_name (TREE_TYPE (view_op0)); |
124 | |
125 | new_stmt = gimple_build_call_internal (IFN_VEC_SET, 3, var_src, |
126 | val, pos); |
127 | |
128 | gimple_call_set_lhs (gs: new_stmt, lhs: var_dst); |
129 | gimple_set_location (g: new_stmt, location: loc); |
130 | gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT); |
131 | |
132 | ass_stmt = gimple_build_assign (view_op0, var_dst); |
133 | gimple_set_location (g: ass_stmt, location: loc); |
134 | gimple_move_vops (ass_stmt, stmt); |
135 | gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT); |
136 | |
137 | basic_block bb = gimple_bb (g: stmt); |
138 | if (gsi_remove (gsi, true) |
139 | && gimple_purge_dead_eh_edges (bb)) |
140 | cfg_changed = true; |
141 | *gsi = gsi_for_stmt (ass_stmt); |
142 | } |
143 | else |
144 | { |
145 | new_stmt |
146 | = gimple_build_call_internal (IFN_VEC_EXTRACT, 2, var_src, pos); |
147 | gimple_call_set_lhs (gs: new_stmt, lhs); |
148 | |
149 | gsi_replace (gsi, new_stmt, true); |
150 | cfg_changed = true; |
151 | } |
152 | } |
153 | } |
154 | |
155 | return cfg_changed; |
156 | } |
157 | |
158 | /* Expand all VEC_COND_EXPR gimple assignments into calls to internal |
159 | function based on type of selected expansion. */ |
160 | |
161 | static gimple * |
162 | gimple_expand_vec_cond_expr (struct function *fun, gimple_stmt_iterator *gsi, |
163 | hash_map<tree, unsigned int> *vec_cond_ssa_name_uses) |
164 | { |
165 | tree lhs, op0a = NULL_TREE, op0b = NULL_TREE; |
166 | enum tree_code code; |
167 | enum tree_code tcode; |
168 | machine_mode cmp_op_mode; |
169 | bool unsignedp; |
170 | enum insn_code icode; |
171 | imm_use_iterator imm_iter; |
172 | |
173 | /* Only consider code == GIMPLE_ASSIGN. */ |
174 | gassign *stmt = dyn_cast<gassign *> (p: gsi_stmt (i: *gsi)); |
175 | if (!stmt) |
176 | return NULL; |
177 | |
178 | code = gimple_assign_rhs_code (gs: stmt); |
179 | if (code != VEC_COND_EXPR) |
180 | return NULL; |
181 | |
182 | tree op0 = gimple_assign_rhs1 (gs: stmt); |
183 | tree op1 = gimple_assign_rhs2 (gs: stmt); |
184 | tree op2 = gimple_assign_rhs3 (gs: stmt); |
185 | lhs = gimple_assign_lhs (gs: stmt); |
186 | machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); |
187 | |
188 | /* Lower mask typed, non-vector mode VEC_COND_EXPRs to bitwise operations. |
189 | Those can end up generated by folding and at least for integer mode masks |
190 | we cannot expect vcond expanders to exist. We lower a ? b : c |
191 | to (b & a) | (c & ~a). */ |
192 | if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)) |
193 | && !VECTOR_MODE_P (mode)) |
194 | { |
195 | gcc_assert (types_compatible_p (TREE_TYPE (op0), TREE_TYPE (op1))); |
196 | gimple_seq stmts = NULL; |
197 | tree type = TREE_TYPE (lhs); |
198 | location_t loc = gimple_location (g: stmt); |
199 | tree tem0 = gimple_build (seq: &stmts, loc, code: BIT_AND_EXPR, type, ops: op1, ops: op0); |
200 | tree tem1 = gimple_build (seq: &stmts, loc, code: BIT_NOT_EXPR, type, ops: op0); |
201 | tree tem2 = gimple_build (seq: &stmts, loc, code: BIT_AND_EXPR, type, ops: op2, ops: tem1); |
202 | tree tem3 = gimple_build (seq: &stmts, loc, code: BIT_IOR_EXPR, type, ops: tem0, ops: tem2); |
203 | gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT); |
204 | return gimple_build_assign (lhs, tem3); |
205 | } |
206 | |
207 | bool can_compute_op0 = true; |
208 | gcc_assert (!COMPARISON_CLASS_P (op0)); |
209 | if (TREE_CODE (op0) == SSA_NAME) |
210 | { |
211 | unsigned int used_vec_cond_exprs = 0; |
212 | unsigned int *slot = vec_cond_ssa_name_uses->get (k: op0); |
213 | if (slot) |
214 | used_vec_cond_exprs = *slot; |
215 | else |
216 | { |
217 | gimple *use_stmt; |
218 | FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, op0) |
219 | { |
220 | gassign *assign = dyn_cast<gassign *> (p: use_stmt); |
221 | if (assign != NULL |
222 | && gimple_assign_rhs_code (gs: assign) == VEC_COND_EXPR |
223 | && gimple_assign_rhs1 (gs: assign) == op0) |
224 | used_vec_cond_exprs++; |
225 | } |
226 | vec_cond_ssa_name_uses->put (k: op0, v: used_vec_cond_exprs); |
227 | } |
228 | |
229 | gassign *def_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op0)); |
230 | if (def_stmt) |
231 | { |
232 | tcode = gimple_assign_rhs_code (gs: def_stmt); |
233 | op0a = gimple_assign_rhs1 (gs: def_stmt); |
234 | op0b = gimple_assign_rhs2 (gs: def_stmt); |
235 | |
236 | tree op0_type = TREE_TYPE (op0); |
237 | tree op0a_type = TREE_TYPE (op0a); |
238 | if (TREE_CODE_CLASS (tcode) == tcc_comparison) |
239 | can_compute_op0 = expand_vec_cmp_expr_p (op0a_type, op0_type, |
240 | tcode); |
241 | |
242 | /* Try to fold x CMP y ? -1 : 0 to x CMP y. */ |
243 | if (can_compute_op0 |
244 | && integer_minus_onep (op1) |
245 | && integer_zerop (op2) |
246 | && TYPE_MODE (TREE_TYPE (lhs)) == TYPE_MODE (TREE_TYPE (op0))) |
247 | { |
248 | tree conv_op = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), op0); |
249 | gassign *new_stmt = gimple_build_assign (lhs, conv_op); |
250 | gsi_replace (gsi, new_stmt, true); |
251 | return new_stmt; |
252 | } |
253 | |
254 | /* When the compare has EH we do not want to forward it when |
255 | it has multiple uses and in general because of the complication |
256 | with EH redirection. */ |
257 | if (stmt_can_throw_internal (fun, def_stmt)) |
258 | tcode = TREE_CODE (op0); |
259 | |
260 | /* If we can compute op0 and have multiple uses, keep the SSA |
261 | name and use vcond_mask. */ |
262 | else if (can_compute_op0 |
263 | && used_vec_cond_exprs >= 2 |
264 | && (get_vcond_mask_icode (vmode: mode, TYPE_MODE (op0_type)) |
265 | != CODE_FOR_nothing)) |
266 | tcode = TREE_CODE (op0); |
267 | } |
268 | else |
269 | tcode = TREE_CODE (op0); |
270 | } |
271 | else |
272 | tcode = TREE_CODE (op0); |
273 | |
274 | if (TREE_CODE_CLASS (tcode) != tcc_comparison) |
275 | { |
276 | gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))); |
277 | if (get_vcond_mask_icode (vmode: mode, TYPE_MODE (TREE_TYPE (op0))) |
278 | != CODE_FOR_nothing) |
279 | return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2); |
280 | /* Fake op0 < 0. */ |
281 | else |
282 | { |
283 | gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0))) |
284 | == MODE_VECTOR_INT); |
285 | op0a = op0; |
286 | op0b = build_zero_cst (TREE_TYPE (op0)); |
287 | tcode = LT_EXPR; |
288 | } |
289 | } |
290 | cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a)); |
291 | unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a)); |
292 | |
293 | gcc_assert (known_eq (GET_MODE_NUNITS (mode), |
294 | GET_MODE_NUNITS (cmp_op_mode))); |
295 | |
296 | icode = get_vcond_icode (vmode: mode, cmode: cmp_op_mode, uns: unsignedp); |
297 | /* Some targets do not have vcondeq and only vcond with NE/EQ |
298 | but not vcondu, so make sure to also try vcond here as |
299 | vcond_icode_p would canonicalize the optab query to. */ |
300 | if (icode == CODE_FOR_nothing |
301 | && (tcode == NE_EXPR || tcode == EQ_EXPR) |
302 | && ((icode = get_vcond_icode (vmode: mode, cmode: cmp_op_mode, uns: !unsignedp)) |
303 | != CODE_FOR_nothing)) |
304 | unsignedp = !unsignedp; |
305 | if (icode == CODE_FOR_nothing) |
306 | { |
307 | if (tcode == LT_EXPR |
308 | && op0a == op0) |
309 | { |
310 | /* A VEC_COND_EXPR condition could be folded from EQ_EXPR/NE_EXPR |
311 | into a constant when only get_vcond_eq_icode is supported. |
312 | Try changing it to NE_EXPR. */ |
313 | tcode = NE_EXPR; |
314 | } |
315 | if ((tcode == EQ_EXPR || tcode == NE_EXPR) |
316 | && direct_internal_fn_supported_p (fn: IFN_VCONDEQ, TREE_TYPE (lhs), |
317 | TREE_TYPE (op0a), |
318 | opt_type: OPTIMIZE_FOR_BOTH)) |
319 | { |
320 | tree tcode_tree = build_int_cst (integer_type_node, tcode); |
321 | return gimple_build_call_internal (IFN_VCONDEQ, 5, op0a, op0b, op1, |
322 | op2, tcode_tree); |
323 | } |
324 | |
325 | gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)) |
326 | && can_compute_op0 |
327 | && (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0))) |
328 | != CODE_FOR_nothing)); |
329 | return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2); |
330 | } |
331 | |
332 | tree tcode_tree = build_int_cst (integer_type_node, tcode); |
333 | return gimple_build_call_internal (unsignedp ? IFN_VCONDU : IFN_VCOND, |
334 | 5, op0a, op0b, op1, op2, tcode_tree); |
335 | } |
336 | |
337 | |
338 | |
339 | namespace { |
340 | |
341 | const pass_data pass_data_gimple_isel = |
342 | { |
343 | .type: GIMPLE_PASS, /* type */ |
344 | .name: "isel" , /* name */ |
345 | .optinfo_flags: OPTGROUP_VEC, /* optinfo_flags */ |
346 | .tv_id: TV_NONE, /* tv_id */ |
347 | PROP_cfg, /* properties_required */ |
348 | .properties_provided: 0, /* properties_provided */ |
349 | .properties_destroyed: 0, /* properties_destroyed */ |
350 | .todo_flags_start: 0, /* todo_flags_start */ |
351 | TODO_update_ssa, /* todo_flags_finish */ |
352 | }; |
353 | |
354 | class pass_gimple_isel : public gimple_opt_pass |
355 | { |
356 | public: |
357 | pass_gimple_isel (gcc::context *ctxt) |
358 | : gimple_opt_pass (pass_data_gimple_isel, ctxt) |
359 | {} |
360 | |
361 | /* opt_pass methods: */ |
362 | bool gate (function *) final override |
363 | { |
364 | return true; |
365 | } |
366 | |
367 | unsigned int execute (function *fun) final override; |
368 | }; // class pass_gimple_isel |
369 | |
370 | |
371 | /* Iterate all gimple statements and perform pre RTL expansion |
372 | GIMPLE massaging to improve instruction selection. */ |
373 | |
374 | unsigned int |
375 | pass_gimple_isel::execute (struct function *fun) |
376 | { |
377 | gimple_stmt_iterator gsi; |
378 | basic_block bb; |
379 | hash_map<tree, unsigned int> vec_cond_ssa_name_uses; |
380 | auto_bitmap dce_ssa_names; |
381 | bool cfg_changed = false; |
382 | |
383 | FOR_EACH_BB_FN (bb, fun) |
384 | { |
385 | for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi)) |
386 | { |
387 | /* Pre-expand VEC_COND_EXPRs to .VCOND* internal function |
388 | calls mapping to supported optabs. */ |
389 | gimple *g = gimple_expand_vec_cond_expr (fun, gsi: &gsi, |
390 | vec_cond_ssa_name_uses: &vec_cond_ssa_name_uses); |
391 | if (g != NULL) |
392 | { |
393 | tree lhs = gimple_assign_lhs (gs: gsi_stmt (i: gsi)); |
394 | gimple_set_lhs (g, lhs); |
395 | gsi_replace (&gsi, g, false); |
396 | } |
397 | |
398 | /* Recognize .VEC_SET and .VEC_EXTRACT patterns. */ |
399 | cfg_changed |= gimple_expand_vec_set_extract_expr (fun, gsi: &gsi); |
400 | if (gsi_end_p (i: gsi)) |
401 | break; |
402 | |
403 | gassign *stmt = dyn_cast <gassign *> (p: *gsi); |
404 | if (!stmt) |
405 | continue; |
406 | |
407 | tree_code code = gimple_assign_rhs_code (gs: stmt); |
408 | tree lhs = gimple_assign_lhs (gs: stmt); |
409 | if (TREE_CODE_CLASS (code) == tcc_comparison |
410 | && !has_single_use (var: lhs)) |
411 | { |
412 | /* Duplicate COND_EXPR condition defs when they are |
413 | comparisons so RTL expansion with the help of TER |
414 | can perform better if conversion. */ |
415 | imm_use_iterator imm_iter; |
416 | use_operand_p use_p; |
417 | auto_vec<gassign *, 4> cond_exprs; |
418 | unsigned cnt = 0; |
419 | FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs) |
420 | { |
421 | if (is_gimple_debug (USE_STMT (use_p))) |
422 | continue; |
423 | cnt++; |
424 | if (gimple_bb (USE_STMT (use_p)) == bb |
425 | && is_gimple_assign (USE_STMT (use_p)) |
426 | && gimple_assign_rhs1_ptr (USE_STMT (use_p)) == use_p->use |
427 | && gimple_assign_rhs_code (USE_STMT (use_p)) == COND_EXPR) |
428 | cond_exprs.safe_push (obj: as_a <gassign *> (USE_STMT (use_p))); |
429 | } |
430 | for (unsigned i = cond_exprs.length () == cnt ? 1 : 0; |
431 | i < cond_exprs.length (); ++i) |
432 | { |
433 | gassign *copy = as_a <gassign *> (p: gimple_copy (stmt)); |
434 | tree new_def = duplicate_ssa_name (var: lhs, stmt: copy); |
435 | gimple_assign_set_lhs (gs: copy, lhs: new_def); |
436 | auto gsi2 = gsi_for_stmt (cond_exprs[i]); |
437 | gsi_insert_before (&gsi2, copy, GSI_SAME_STMT); |
438 | gimple_assign_set_rhs1 (gs: cond_exprs[i], rhs: new_def); |
439 | update_stmt (s: cond_exprs[i]); |
440 | } |
441 | } |
442 | } |
443 | } |
444 | |
445 | for (auto it = vec_cond_ssa_name_uses.begin (); |
446 | it != vec_cond_ssa_name_uses.end (); ++it) |
447 | bitmap_set_bit (dce_ssa_names, SSA_NAME_VERSION ((*it).first)); |
448 | |
449 | simple_dce_from_worklist (dce_ssa_names); |
450 | |
451 | return cfg_changed ? TODO_cleanup_cfg : 0; |
452 | } |
453 | |
454 | } // anon namespace |
455 | |
456 | gimple_opt_pass * |
457 | make_pass_gimple_isel (gcc::context *ctxt) |
458 | { |
459 | return new pass_gimple_isel (ctxt); |
460 | } |
461 | |
462 | |