diff options
Diffstat (limited to 'gcc/graphite.c')
| -rw-r--r-- | gcc/graphite.c | 1552 |
1 files changed, 844 insertions, 708 deletions
diff --git a/gcc/graphite.c b/gcc/graphite.c index 78485a2c833..7c2b8da90b9 100644 --- a/gcc/graphite.c +++ b/gcc/graphite.c @@ -51,6 +51,7 @@ along with GCC; see the file COPYING3. If not see #include "tree-scalar-evolution.h" #include "tree-pass.h" #include "domwalk.h" +#include "value-prof.h" #include "pointer-set.h" #include "gimple.h" @@ -167,13 +168,9 @@ static tree loop_iv_stack_get_iv (loop_iv_stack stack, int index) { iv_stack_entry_p entry = VEC_index (iv_stack_entry_p, *stack, index); + iv_stack_entry_data data = entry->data; - tree result = NULL; - - if (entry->kind != iv_stack_entry_const) - result = entry->data.iv->t; - - return result; + return iv_stack_entry_is_iv (entry) ? data.iv->t : data.constant; } /* Get the IV from its NAME in STACK. */ @@ -293,33 +290,6 @@ loop_iv_stack_remove_constants (loop_iv_stack stack) } } -/* In SCOP, get the induction variable from NAME. OLD is the original - loop that contained the definition of NAME. */ - -static name_tree -get_old_iv_from_ssa_name (scop_p scop, loop_p old, tree name) -{ - tree var = SSA_NAME_VAR (name); - int i; - name_tree oldiv; - - for (i = 0; VEC_iterate (name_tree, SCOP_OLDIVS (scop), i, oldiv); i++) - { - loop_p current = old; - - while (current) - { - if (var == oldiv->t - && oldiv->loop == current) - return oldiv; - - current = loop_outer (current); - } - } - return NULL; - -} - /* Returns a new loop_to_cloog_loop_str structure. */ static inline struct loop_to_cloog_loop_str * @@ -610,14 +580,6 @@ debug_scops (int verbosity) print_scops (stderr, verbosity); } -/* Return true when BB is contained in SCOP. */ - -static inline bool -bb_in_scop_p (basic_block bb, scop_p scop) -{ - return bitmap_bit_p (SCOP_BBS_B (scop), bb->index); -} - /* Pretty print to FILE the SCOP in DOT format. */ static void @@ -818,15 +780,6 @@ dot_all_scops (void) #endif } -/* Returns true when LOOP is in SCOP. */ - -static inline bool -loop_in_scop_p (struct loop *loop, scop_p scop) -{ - return (bb_in_scop_p (loop->header, scop) - && bb_in_scop_p (loop->latch, scop)); -} - /* Returns the outermost loop in SCOP that contains BB. */ static struct loop * @@ -1025,23 +978,85 @@ harmful_stmt_in_bb (basic_block scop_entry, basic_block bb) return NULL; } +/* Returns true when BB will be represented in graphite. Return false + for the basic blocks that contain code eliminated in the code + generation pass: i.e. induction variables and exit conditions. */ + +static bool +graphite_stmt_p (scop_p scop, basic_block bb, + VEC (data_reference_p, heap) *drs) +{ + gimple_stmt_iterator gsi; + loop_p loop = bb->loop_father; + + if (VEC_length (data_reference_p, drs) > 0) + return true; + + for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) + { + gimple stmt = gsi_stmt (gsi); + + switch (gimple_code (stmt)) + { + /* Control flow expressions can be ignored, as they are + represented in the iteration domains and will be + regenerated by graphite. */ + case GIMPLE_COND: + case GIMPLE_GOTO: + case GIMPLE_SWITCH: + break; + + case GIMPLE_ASSIGN: + { + tree var = gimple_assign_lhs (stmt); + var = analyze_scalar_evolution (loop, var); + var = instantiate_scev (block_before_scop (scop), loop, var); + + if (chrec_contains_undetermined (var)) + return true; + + break; + } + + default: + return true; + } + } + + return false; +} + /* Store the GRAPHITE representation of BB. */ static void new_graphite_bb (scop_p scop, basic_block bb) { - struct graphite_bb *gbb = XNEW (struct graphite_bb); + struct graphite_bb *gbb; + VEC (data_reference_p, heap) *drs = VEC_alloc (data_reference_p, heap, 5); + struct loop *nest = outermost_loop_in_scop (scop, bb); + gimple_stmt_iterator gsi; + + bitmap_set_bit (SCOP_BBS_B (scop), bb->index); + for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) + find_data_references_in_stmt (nest, gsi_stmt (gsi), &drs); + + if (!graphite_stmt_p (scop, bb, drs)) + { + free_data_refs (drs); + return; + } + + gbb = XNEW (struct graphite_bb); bb->aux = gbb; GBB_BB (gbb) = bb; GBB_SCOP (gbb) = scop; - GBB_DATA_REFS (gbb) = NULL; + GBB_DATA_REFS (gbb) = drs; GBB_DOMAIN (gbb) = NULL; GBB_CONDITIONS (gbb) = NULL; GBB_CONDITION_CASES (gbb) = NULL; GBB_LOOPS (gbb) = NULL; VEC_safe_push (graphite_bb_p, heap, SCOP_BBS (scop), gbb); - bitmap_set_bit (SCOP_BBS_B (scop), bb->index); } /* Frees GBB. */ @@ -1052,11 +1067,9 @@ free_graphite_bb (struct graphite_bb *gbb) if (GBB_DOMAIN (gbb)) cloog_matrix_free (GBB_DOMAIN (gbb)); - free_data_refs (GBB_DATA_REFS (gbb)); VEC_free (gimple, heap, GBB_CONDITIONS (gbb)); VEC_free (gimple, heap, GBB_CONDITION_CASES (gbb)); VEC_free (loop_p, heap, GBB_LOOPS (gbb)); - GBB_BB (gbb)->aux = 0; XDELETE (gbb); } @@ -1078,6 +1091,7 @@ new_scop (edge entry, edge exit) SCOP_BBS_B (scop) = BITMAP_ALLOC (NULL); SCOP_LOOPS (scop) = BITMAP_ALLOC (NULL); SCOP_LOOP_NEST (scop) = VEC_alloc (loop_p, heap, 3); + SCOP_ADD_PARAMS (scop) = true; SCOP_PARAMS (scop) = VEC_alloc (name_tree, heap, 3); SCOP_PROG (scop) = cloog_program_malloc (); cloog_program_set_names (SCOP_PROG (scop), cloog_names_malloc ()); @@ -1511,6 +1525,8 @@ build_scops_1 (basic_block current, VEC (sd_region, heap) **scops, loop_p loop) result.next = NULL; result.last = NULL; open_scop.entry = NULL; + open_scop.exit = NULL; + sinfo.last = NULL; /* Loop over the dominance tree. If we meet a difficult bb, close the current SCoP. Loop and condition header start a new layer, @@ -1799,6 +1815,29 @@ mark_exit_edges (VEC (sd_region, heap) *regions) e->aux = s; } +/* Free and compute again all the dominators information. */ + +static inline void +recompute_all_dominators (void) +{ + free_dominance_info (CDI_DOMINATORS); + free_dominance_info (CDI_POST_DOMINATORS); + calculate_dominance_info (CDI_DOMINATORS); + calculate_dominance_info (CDI_POST_DOMINATORS); +} + +/* Verifies properties that GRAPHITE should maintain during translation. */ + +static inline void +graphite_verify (void) +{ +#ifdef ENABLE_CHECKING + verify_loop_structure (); + verify_dominators (CDI_DOMINATORS); + verify_dominators (CDI_POST_DOMINATORS); + verify_ssa (false); +#endif +} /* Create for all scop regions a single entry and a single exit edge. */ @@ -1937,64 +1976,108 @@ build_scop_bbs (scop_p scop) sbitmap_free (visited); } +/* Returns the number of reduction phi nodes in LOOP. */ -/* Record LOOP as occuring in SCOP. */ +static int +nb_reductions_in_loop (loop_p loop) +{ + int res = 0; + gimple_stmt_iterator gsi; -static void -scop_record_loop (scop_p scop, struct loop *loop) + for (gsi = gsi_start_phis (loop->header); !gsi_end_p (gsi); gsi_next (&gsi)) + { + gimple phi = gsi_stmt (gsi); + tree scev; + + if (!is_gimple_reg (PHI_RESULT (phi))) + continue; + + scev = analyze_scalar_evolution (loop, PHI_RESULT (phi)); + scev = instantiate_parameters (loop, scev); + if (chrec_contains_undetermined (scev)) + res++; + } + + return res; +} + +/* A LOOP is in normal form when it contains only one scalar phi node + that defines the main induction variable of the loop, only one + increment of the IV, and only one exit condition. */ + +static tree +graphite_loop_normal_form (loop_p loop) { - loop_p parent; - tree induction_var; + struct tree_niter_desc niter; + tree nit; + gimple_seq stmts; + edge exit = single_dom_exit (loop); - if (bitmap_bit_p (SCOP_LOOPS (scop), loop->num)) - return; + if (!number_of_iterations_exit (loop, exit, &niter, false)) + gcc_unreachable (); - parent = loop_outer (loop); - induction_var = find_induction_var_from_exit_cond (loop); + nit = force_gimple_operand (unshare_expr (niter.niter), &stmts, true, + NULL_TREE); + if (stmts) + gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts); - if (!bb_in_scop_p (parent->latch, scop)) - parent = NULL; + /* One IV per loop. */ + if (nb_reductions_in_loop (loop) > 0) + return NULL_TREE; - if (induction_var != NULL_TREE) - { - name_tree oldiv = XNEW (struct name_tree); - oldiv->t = SSA_NAME_VAR (induction_var); - if (DECL_NAME (oldiv->t)) - oldiv->name = IDENTIFIER_POINTER (DECL_NAME (oldiv->t)); - else - { - int len = 2 + 16; - char *n = XNEWVEC (char, len); - snprintf (n, len, "D.%u", DECL_UID (oldiv->t)); - oldiv->name = n; - } - oldiv->loop = loop; + return canonicalize_loop_ivs (loop, NULL, nit); +} - VEC_safe_push (name_tree, heap, SCOP_OLDIVS (scop), oldiv); - } +/* Record LOOP as occuring in SCOP. Returns true when the operation + was successful. */ + +static bool +scop_record_loop (scop_p scop, loop_p loop) +{ + tree induction_var; + name_tree oldiv; + + if (bitmap_bit_p (SCOP_LOOPS (scop), loop->num)) + return true; bitmap_set_bit (SCOP_LOOPS (scop), loop->num); VEC_safe_push (loop_p, heap, SCOP_LOOP_NEST (scop), loop); + + induction_var = graphite_loop_normal_form (loop); + if (!induction_var) + return false; + + oldiv = XNEW (struct name_tree); + oldiv->t = induction_var; + oldiv->name = get_name (SSA_NAME_VAR (oldiv->t)); + oldiv->loop = loop; + VEC_safe_push (name_tree, heap, SCOP_OLDIVS (scop), oldiv); + return true; } -/* Build the loop nests contained in SCOP. */ +/* Build the loop nests contained in SCOP. Returns true when the + operation was successful. */ -static void +static bool build_scop_loop_nests (scop_p scop) { unsigned i; - graphite_bb_p gb; + basic_block bb; struct loop *loop0, *loop1; - for (i = 0; VEC_iterate (graphite_bb_p, SCOP_BBS (scop), i, gb); i++) - { - struct loop *loop = gbb_loop (gb); + FOR_EACH_BB (bb) + if (bb_in_scop_p (bb, scop)) + { + struct loop *loop = bb->loop_father; - /* Only add loops, if they are completely contained in the SCoP. */ - if (loop->header == GBB_BB (gb) - && bb_in_scop_p (loop->latch, scop)) - scop_record_loop (scop, gbb_loop (gb)); - } + /* Only add loops if they are completely contained in the SCoP. */ + if (loop->header == bb + && bb_in_scop_p (loop->latch, scop)) + { + if (!scop_record_loop (scop, loop)) + return false; + } + } /* Make sure that the loops in the SCOP_LOOP_NEST are ordered. It can be the case that an inner loop is inserted before an outer @@ -2011,20 +2094,37 @@ build_scop_loop_nests (scop_p scop) VEC_replace (loop_p, SCOP_LOOP_NEST (scop), i + 1, loop0); } } + + return true; } -/* Calculate the number of loops around GB in the current SCOP. */ +/* Build dynamic schedules for all the BBs. */ -static inline int -nb_loops_around_gb (graphite_bb_p gb) +static void +build_scop_dynamic_schedules (scop_p scop) { - scop_p scop = GBB_SCOP (gb); - struct loop *l = gbb_loop (gb); - int d = 0; + int i, dim, loop_num, row, col; + graphite_bb_p gb; - for (; loop_in_scop_p (l, scop); d++, l = loop_outer (l)); + for (i = 0; VEC_iterate (graphite_bb_p, SCOP_BBS (scop), i, gb); i++) + { + loop_num = GBB_BB (gb)->loop_father->num; - return d; + if (loop_num != 0) + { + dim = nb_loops_around_gb (gb); + GBB_DYNAMIC_SCHEDULE (gb) = cloog_matrix_alloc (dim, dim); + + for (row = 0; row < GBB_DYNAMIC_SCHEDULE (gb)->NbRows; row++) + for (col = 0; col < GBB_DYNAMIC_SCHEDULE (gb)->NbColumns; col++) + if (row == col) + value_set_si (GBB_DYNAMIC_SCHEDULE (gb)->p[row][col], 1); + else + value_set_si (GBB_DYNAMIC_SCHEDULE (gb)->p[row][col], 0); + } + else + GBB_DYNAMIC_SCHEDULE (gb) = NULL; + } } /* Build for BB the static schedule. @@ -2138,6 +2238,8 @@ param_index (tree var, scop_p scop) if (p->t == var) return i; + gcc_assert (SCOP_ADD_PARAMS (scop)); + nvar = XNEW (struct name_tree); nvar->t = var; nvar->name = NULL; @@ -2217,26 +2319,28 @@ scan_tree_for_params (scop_p s, tree e, CloogMatrix *c, int r, Value k, case MULT_EXPR: if (chrec_contains_symbols (TREE_OPERAND (e, 0))) { - Value val; - - gcc_assert (host_integerp (TREE_OPERAND (e, 1), 0)); - - value_init (val); - value_set_si (val, int_cst_value (TREE_OPERAND (e, 1))); - value_multiply (k, k, val); - value_clear (val); + if (c) + { + Value val; + gcc_assert (host_integerp (TREE_OPERAND (e, 1), 0)); + value_init (val); + value_set_si (val, int_cst_value (TREE_OPERAND (e, 1))); + value_multiply (k, k, val); + value_clear (val); + } scan_tree_for_params (s, TREE_OPERAND (e, 0), c, r, k, subtract); } else { - Value val; - - gcc_assert (host_integerp (TREE_OPERAND (e, 0), 0)); - - value_init (val); - value_set_si (val, int_cst_value (TREE_OPERAND (e, 0))); - value_multiply (k, k, val); - value_clear (val); + if (c) + { + Value val; + gcc_assert (host_integerp (TREE_OPERAND (e, 0), 0)); + value_init (val); + value_set_si (val, int_cst_value (TREE_OPERAND (e, 0))); + value_multiply (k, k, val); + value_clear (val); + } scan_tree_for_params (s, TREE_OPERAND (e, 1), c, r, k, subtract); } break; @@ -2347,58 +2451,44 @@ idx_record_params (tree base, tree *idx, void *dta) access functions, conditions and loop bounds. */ static void -find_params_in_bb (scop_p scop, basic_block bb) +find_params_in_bb (scop_p scop, graphite_bb_p gb) { int i; data_reference_p dr; - VEC (data_reference_p, heap) *drs; - gimple_stmt_iterator gsi; - struct loop *nest = outermost_loop_in_scop (scop, bb); - - /* Find the parameters used in the memory access functions. */ - drs = VEC_alloc (data_reference_p, heap, 5); - for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) - find_data_references_in_stmt (nest, gsi_stmt (gsi), &drs); + gimple stmt; + loop_p father = GBB_BB (gb)->loop_father; - for (i = 0; VEC_iterate (data_reference_p, drs, i, dr); i++) + for (i = 0; VEC_iterate (data_reference_p, GBB_DATA_REFS (gb), i, dr); i++) { struct irp_data irp; - irp.loop = bb->loop_father; + irp.loop = father; irp.scop = scop; for_each_index (&dr->ref, idx_record_params, &irp); free_data_ref (dr); } - VEC_free (data_reference_p, heap, drs); - /* Find parameters in conditional statements. */ - gsi = gsi_last_bb (bb); - if (!gsi_end_p (gsi)) + for (i = 0; VEC_iterate (gimple, GBB_CONDITIONS (gb), i, stmt); i++) { - gimple stmt = gsi_stmt (gsi); + Value one; + loop_p loop = father; - if (gimple_code (stmt) == GIMPLE_COND) - { - Value one; - loop_p loop = bb->loop_father; - - tree lhs, rhs; - - lhs = gimple_cond_lhs (stmt); - lhs = analyze_scalar_evolution (loop, lhs); - lhs = instantiate_scev (block_before_scop (scop), loop, lhs); - - rhs = gimple_cond_rhs (stmt); - rhs = analyze_scalar_evolution (loop, rhs); - rhs = instantiate_scev (block_before_scop (scop), loop, rhs); - - value_init (one); - scan_tree_for_params (scop, lhs, NULL, 0, one, false); - value_set_si (one, 1); - scan_tree_for_params (scop, rhs, NULL, 0, one, false); - value_clear (one); - } + tree lhs, rhs; + + lhs = gimple_cond_lhs (stmt); + lhs = analyze_scalar_evolution (loop, lhs); + lhs = instantiate_scev (block_before_scop (scop), loop, lhs); + + rhs = gimple_cond_rhs (stmt); + rhs = analyze_scalar_evolution (loop, rhs); + rhs = instantiate_scev (block_before_scop (scop), loop, rhs); + + value_init (one); + scan_tree_for_params (scop, lhs, NULL, 0, one, false); + value_set_si (one, 1); + scan_tree_for_params (scop, rhs, NULL, 0, one, false); + value_clear (one); } } @@ -2522,7 +2612,9 @@ find_scop_parameters (scop_p scop) /* Find the parameters used in data accesses. */ for (i = 0; VEC_iterate (graphite_bb_p, SCOP_BBS (scop), i, gb); i++) - find_params_in_bb (scop, GBB_BB (gb)); + find_params_in_bb (scop, gb); + + SCOP_ADD_PARAMS (scop) = false; } /* Build the context constraints for SCOP: constraints and relations @@ -2554,24 +2646,6 @@ gbb_from_bb (basic_block bb) return (graphite_bb_p) bb->aux; } -/* Add DOMAIN to all the basic blocks in LOOP. */ - -static void -add_bb_domains (struct loop *loop, CloogMatrix *domain) -{ - basic_block *bbs = get_loop_body (loop); - unsigned i; - - for (i = 0; i < loop->num_nodes; i++) - if (bbs[i]->loop_father == loop) - { - graphite_bb_p gbb = gbb_from_bb (bbs[i]); - GBB_DOMAIN (gbb) = cloog_matrix_copy (domain); - } - - free (bbs); -} - /* Builds the constraint matrix for LOOP in SCOP. NB_OUTER_LOOPS is the number of loops surrounding LOOP in SCOP. OUTER_CSTR gives the constraints matrix for the surrounding loops. */ @@ -2582,6 +2656,7 @@ build_loop_iteration_domains (scop_p scop, struct loop *loop, { int i, j, row; CloogMatrix *cstr; + graphite_bb_p gb; int nb_rows = outer_cstr->NbRows + 1; int nb_cols = outer_cstr->NbColumns + 1; @@ -2662,7 +2737,9 @@ build_loop_iteration_domains (scop_p scop, struct loop *loop, if (nb_outer_loops != 0 && loop->next && loop_in_scop_p (loop->next, scop)) build_loop_iteration_domains (scop, loop->next, outer_cstr, nb_outer_loops); - add_bb_domains (loop, cstr); + for (i = 0; VEC_iterate (graphite_bb_p, SCOP_BBS (scop), i, gb); i++) + if (gbb_loop (gb) == loop) + GBB_DOMAIN (gb) = cloog_matrix_copy (cstr); cloog_matrix_free (cstr); } @@ -2884,10 +2961,12 @@ build_scop_conditions_1 (VEC (gimple, heap) **conditions, /* Record conditions in graphite_bb. */ gbb = gbb_from_bb (bb); - GBB_CONDITIONS (gbb) = VEC_copy (gimple, heap, *conditions); - GBB_CONDITION_CASES (gbb) = VEC_copy (gimple, heap, *cases); - - add_conditions_to_domain (gbb); + if (gbb) + { + GBB_CONDITIONS (gbb) = VEC_copy (gimple, heap, *conditions); + GBB_CONDITION_CASES (gbb) = VEC_copy (gimple, heap, *cases); + add_conditions_to_domain (gbb); + } dom = get_dominated_by (CDI_DOMINATORS, bb); @@ -3049,27 +3128,31 @@ build_scop_iteration_domain (scop_p scop) } /* Initializes an equation CY of the access matrix using the - information for a subscript from ACCESS_FUN, relatively to the loop + information for a subscript from AF, relatively to the loop indexes from LOOP_NEST and parameter indexes from PARAMS. NDIM is the dimension of the array access, i.e. the number of subscripts. Returns true when the operation succeeds. */ static bool -build_access_matrix_with_af (tree access_fun, lambda_vector cy, +build_access_matrix_with_af (tree af, lambda_vector cy, scop_p scop, int ndim) { - switch (TREE_CODE (access_fun)) + int param_col; + + switch (TREE_CODE (af)) { case POLYNOMIAL_CHREC: { - tree left = CHREC_LEFT (access_fun); - tree right = CHREC_RIGHT (access_fun); + struct loop *outer_loop; + tree left = CHREC_LEFT (af); + tree right = CHREC_RIGHT (af); int var; if (TREE_CODE (right) != INTEGER_CST) return false; - - var = loop_iteration_vector_dim (CHREC_VARIABLE (access_fun), scop); + + outer_loop = get_loop (CHREC_VARIABLE (af)); + var = nb_loops_around_loop_in_scop (outer_loop, scop); cy[var] = int_cst_value (right); switch (TREE_CODE (left)) @@ -3082,12 +3165,27 @@ build_access_matrix_with_af (tree access_fun, lambda_vector cy, return true; default: - /* FIXME: access_fn can have parameters. */ - return false; + return build_access_matrix_with_af (left, cy, scop, ndim); } } + + case PLUS_EXPR: + build_access_matrix_with_af (TREE_OPERAND (af, 0), cy, scop, ndim); + build_access_matrix_with_af (TREE_OPERAND (af, 1), cy, scop, ndim); + return true; + + case MINUS_EXPR: + build_access_matrix_with_af (TREE_OPERAND (af, 0), cy, scop, ndim); + build_access_matrix_with_af (TREE_OPERAND (af, 1), cy, scop, ndim); + return true; + case INTEGER_CST: - cy[ndim - 1] = int_cst_value (access_fun); + cy[ndim - 1] = int_cst_value (af); + return true; + + case SSA_NAME: + param_col = param_index (af, scop); + cy [ndim - scop_nb_params (scop) + param_col - 1] = 1; return true; default: @@ -3133,23 +3231,14 @@ build_scop_data_accesses (scop_p scop) int i; graphite_bb_p gb; + /* FIXME: Construction of access matrix is disabled until some + pass, like the data dependence analysis, is using it. */ + return; + for (i = 0; VEC_iterate (graphite_bb_p, SCOP_BBS (scop), i, gb); i++) { int j; - gimple_stmt_iterator gsi; data_reference_p dr; - struct loop *nest = outermost_loop_in_scop (scop, GBB_BB (gb)); - - /* On each statement of the basic block, gather all the occurences - to read/write memory. */ - GBB_DATA_REFS (gb) = VEC_alloc (data_reference_p, heap, 5); - for (gsi = gsi_start_bb (GBB_BB (gb)); !gsi_end_p (gsi); gsi_next (&gsi)) - find_data_references_in_stmt (nest, gsi_stmt (gsi), - &GBB_DATA_REFS (gb)); - - /* FIXME: Construction of access matrix is disabled until some - pass, like the data dependence analysis, is using it. */ - continue; /* Construct the access matrix for each data ref, with respect to the loop nest of the current BB in the considered SCOP. */ @@ -3199,6 +3288,13 @@ clast_name_to_gcc (const char *name, VEC (name_tree, heap) *params, gcc_unreachable (); } +/* A union needed to convert from CLAST expressions to GMP values. */ + +typedef union { + struct clast_expr *c; + Value v; +} value_clast; + /* Converts a Cloog AST expression E back to a GCC expression tree. */ static tree @@ -3299,15 +3395,12 @@ clast_to_gcc_expression (struct clast_expr *e, { struct clast_binary *b = (struct clast_binary *) e; struct clast_expr *lhs = (struct clast_expr *) b->LHS; - struct clast_expr *rhs = (struct clast_expr *) b->RHS; tree tl = clast_to_gcc_expression (lhs, params, ivstack); + value_clast r; + tree tr; - /* FIXME: The next statement produces a warning: Cloog assumes - that the RHS is a constant, but this is a "void *" pointer - that should be casted into a Value, but this cast cannot be - done as Value is a GMP type, that is an array. Cloog must - be fixed for removing this warning. */ - tree tr = gmp_cst_to_tree (rhs); + r.c = (struct clast_expr *) b->RHS; + tr = gmp_cst_to_tree (r.v); switch (b->type) { @@ -3426,40 +3519,98 @@ graphite_create_new_loop (scop_p scop, edge entry_edge, return loop; } -/* Remove all the edges from EDGES except the edge KEEP. */ +/* Structure containing the mapping between the old names and the new + names used after block copy in the new loop context. */ +typedef struct rename_map_elt +{ + tree old_name, new_name; +} *rename_map_elt; + + +/* Print to stderr the element ELT. */ static void -remove_all_edges_1 (VEC (edge, gc) *edges, edge keep) +debug_rename_elt (rename_map_elt elt) { - edge e; - edge_iterator ei; + fprintf (stderr, "("); + print_generic_expr (stderr, elt->old_name, 0); + fprintf (stderr, ", "); + print_generic_expr (stderr, elt->new_name, 0); + fprintf (stderr, ")\n"); +} - for (ei = ei_start (edges); (e = ei_safe_edge (ei)); ) - { - if (e != keep) - { - remove_edge (e); - e = ei_safe_edge (ei); - } - else - ei_next (&ei); - } +/* Helper function for debug_rename_map. */ + +static int +debug_rename_map_1 (void **slot, void *s ATTRIBUTE_UNUSED) +{ + struct rename_map_elt *entry = (struct rename_map_elt *) *slot; + debug_rename_elt (entry); + return 1; } -/* Remove all the edges from BB except the edge KEEP. */ +/* Print to stderr all the elements of MAP. */ -static void -remove_all_edges (basic_block bb, edge keep) +void +debug_rename_map (htab_t map) { - remove_all_edges_1 (bb->succs, keep); - remove_all_edges_1 (bb->preds, keep); + htab_traverse (map, debug_rename_map_1, NULL); +} + +/* Constructs a new SCEV_INFO_STR structure for VAR and INSTANTIATED_BELOW. */ + +static inline rename_map_elt +new_rename_map_elt (tree old_name, tree new_name) +{ + rename_map_elt res; + + res = XNEW (struct rename_map_elt); + res->old_name = old_name; + res->new_name = new_name; + + return res; +} + +/* Computes a hash function for database element ELT. */ + +static hashval_t +rename_map_elt_info (const void *elt) +{ + return htab_hash_pointer (((const struct rename_map_elt *) elt)->old_name); +} + +/* Compares database elements E1 and E2. */ + +static int +eq_rename_map_elts (const void *e1, const void *e2) +{ + const struct rename_map_elt *elt1 = (const struct rename_map_elt *) e1; + const struct rename_map_elt *elt2 = (const struct rename_map_elt *) e2; + + return (elt1->old_name == elt2->old_name); +} + +/* Returns the new name associated to OLD_NAME in MAP. */ + +static tree +get_new_name_from_old_name (htab_t map, tree old_name) +{ + struct rename_map_elt tmp; + PTR *slot; + + tmp.old_name = old_name; + slot = htab_find_slot (map, &tmp, NO_INSERT); + + if (slot && *slot) + return ((rename_map_elt) *slot)->new_name; + + return old_name; } /* Rename the SSA_NAMEs used in STMT and that appear in IVSTACK. */ static void -graphite_rename_ivs_stmt (gimple stmt, graphite_bb_p gbb, scop_p scop, - loop_p old, loop_iv_stack ivstack) +rename_variables_in_stmt (gimple stmt, htab_t map) { ssa_op_iter iter; use_operand_p use_p; @@ -3467,16 +3618,12 @@ graphite_rename_ivs_stmt (gimple stmt, graphite_bb_p gbb, scop_p scop, FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE) { tree use = USE_FROM_PTR (use_p); - tree new_iv = NULL; - name_tree old_iv = get_old_iv_from_ssa_name (scop, old, use); - - if (old_iv) - new_iv = loop_iv_stack_get_iv (ivstack, - gbb_loop_index (gbb, old_iv->loop)); + tree new_name = get_new_name_from_old_name (map, use); - if (new_iv) - SET_USE (use_p, new_iv); + replace_exp (use_p, new_name); } + + update_stmt (stmt); } /* Returns true if SSA_NAME is a parameter of SCOP. */ @@ -3495,28 +3642,26 @@ is_parameter (scop_p scop, tree ssa_name) return false; } -/* Returns true if NAME is an old induction variable in SCOP. OLD is - the original loop that contained the definition of NAME. */ +/* Returns true if NAME is an induction variable. */ static bool -is_old_iv (scop_p scop, loop_p old, tree name) +is_iv (tree name) { - return get_old_iv_from_ssa_name (scop, old, name) != NULL; - + return gimple_code (SSA_NAME_DEF_STMT (name)) == GIMPLE_PHI; } -static void expand_scalar_variables_stmt (gimple, graphite_bb_p, scop_p, loop_p, - loop_iv_stack); +static void expand_scalar_variables_stmt (gimple, basic_block, scop_p, + loop_p, htab_t); /* Constructs a tree which only contains old_ivs and parameters. Any - other variables that are defined outside GBB will be eliminated by + other variables that are defined outside BB will be eliminated by using their definitions in the constructed tree. OLD_LOOP_FATHER - is the original loop that contained GBB. */ + is the original loop that contained BB. */ static tree expand_scalar_variables_expr (tree type, tree op0, enum tree_code code, - tree op1, graphite_bb_p gbb, scop_p scop, - loop_p old_loop_father, loop_iv_stack ivstack) + tree op1, basic_block bb, scop_p scop, + loop_p old_loop_father, htab_t map) { if ((TREE_CODE_CLASS (code) == tcc_constant && code == INTEGER_CST) @@ -3529,8 +3674,7 @@ expand_scalar_variables_expr (tree type, tree op0, enum tree_code code, enum tree_code op0_code = TREE_CODE (op0); tree op0_expr = expand_scalar_variables_expr (op0_type, op0, op0_code, - NULL, gbb, scop, old_loop_father, - ivstack); + NULL, bb, scop, old_loop_father, map); return fold_build1 (code, type, op0_expr); } @@ -3541,14 +3685,12 @@ expand_scalar_variables_expr (tree type, tree op0, enum tree_code code, enum tree_code op0_code = TREE_CODE (op0); tree op0_expr = expand_scalar_variables_expr (op0_type, op0, op0_code, - NULL, gbb, scop, old_loop_father, - ivstack); + NULL, bb, scop, old_loop_father, map); tree op1_type = TREE_TYPE (op1); enum tree_code op1_code = TREE_CODE (op1); tree op1_expr = expand_scalar_variables_expr (op1_type, op1, op1_code, - NULL, gbb, scop, old_loop_father, - ivstack); + NULL, bb, scop, old_loop_father, map); return fold_build2 (code, type, op0_expr, op1_expr); } @@ -3559,34 +3701,34 @@ expand_scalar_variables_expr (tree type, tree op0, enum tree_code code, gimple def_stmt; enum tree_code subcode; - if(is_parameter (scop, op0) || - is_old_iv (scop, old_loop_father, op0)) - return op0; + if (is_parameter (scop, op0) + || is_iv (op0)) + return get_new_name_from_old_name (map, op0); def_stmt = SSA_NAME_DEF_STMT (op0); - if (gimple_bb (def_stmt) == GBB_BB (gbb)) + if (gimple_bb (def_stmt) == bb) { /* If the defining statement is in the basic block already we do not need to create a new expression for it, we only need to ensure its operands are expanded. */ - expand_scalar_variables_stmt (def_stmt, gbb, scop, - old_loop_father, ivstack); - return op0; + expand_scalar_variables_stmt (def_stmt, bb, scop, + old_loop_father, map); + return get_new_name_from_old_name (map, op0); } else { - if (gimple_code (def_stmt) != GIMPLE_ASSIGN) - return op0; + if (gimple_code (def_stmt) != GIMPLE_ASSIGN + || !bb_in_scop_p (gimple_bb (def_stmt), scop)) + return get_new_name_from_old_name (map, op0); var0 = gimple_assign_rhs1 (def_stmt); subcode = gimple_assign_rhs_code (def_stmt); var1 = gimple_assign_rhs2 (def_stmt); - return expand_scalar_variables_expr (type, var0, subcode, var1, - gbb, scop, old_loop_father, - ivstack); + return expand_scalar_variables_expr (type, var0, subcode, var1, + bb, scop, old_loop_father, map); } } @@ -3595,106 +3737,66 @@ expand_scalar_variables_expr (tree type, tree op0, enum tree_code code, } /* Replicates any uses of non-parameters and non-old-ivs variablesthat - are defind outside GBB with code that is inserted in GBB. + are defind outside BB with code that is inserted in BB. OLD_LOOP_FATHER is the original loop that contained STMT. */ static void -expand_scalar_variables_stmt (gimple stmt, graphite_bb_p gbb, scop_p scop, - loop_p old_loop_father, loop_iv_stack ivstack) +expand_scalar_variables_stmt (gimple stmt, basic_block bb, scop_p scop, + loop_p old_loop_father, htab_t map) { ssa_op_iter iter; use_operand_p use_p; - basic_block bb = GBB_BB (gbb); FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE) { tree use = USE_FROM_PTR (use_p); tree type = TREE_TYPE (use); enum tree_code code = TREE_CODE (use); - tree use_expr = expand_scalar_variables_expr (type, use, code, NULL, - gbb, scop, old_loop_father, - ivstack); + tree use_expr = expand_scalar_variables_expr (type, use, code, NULL, bb, + scop, old_loop_father, map); if (use_expr != use) { gimple_stmt_iterator gsi = gsi_after_labels (bb); tree new_use = force_gimple_operand_gsi (&gsi, use_expr, true, NULL, true, GSI_NEW_STMT); - SET_USE (use_p, new_use); + replace_exp (use_p, new_use); } } + + update_stmt (stmt); } -/* Copies the definitions outside of GBB of variables that are not - induction variables nor parameters. GBB must only contain +/* Copies the definitions outside of BB of variables that are not + induction variables nor parameters. BB must only contain "external" references to these types of variables. OLD_LOOP_FATHER - is the original loop that contained GBB. */ + is the original loop that contained BB. */ static void -expand_scalar_variables (graphite_bb_p gbb, scop_p scop, - loop_p old_loop_father, loop_iv_stack ivstack) +expand_scalar_variables (basic_block bb, scop_p scop, + loop_p old_loop_father, htab_t map) { - basic_block bb = GBB_BB (gbb); gimple_stmt_iterator gsi; for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi);) { gimple stmt = gsi_stmt (gsi); - expand_scalar_variables_stmt (stmt, gbb, scop, old_loop_father, - ivstack); + expand_scalar_variables_stmt (stmt, bb, scop, old_loop_father, map); gsi_next (&gsi); } } -/* Rename all the SSA_NAMEs from block GBB that appear in IVSTACK in - terms of new induction variables. OLD_LOOP_FATHER is the original - loop that contained GBB. */ +/* Rename all the SSA_NAMEs from block BB that appear in IVSTACK in + terms of new induction variables. OLD is the original loop that + contained BB. */ static void -graphite_rename_ivs (graphite_bb_p gbb, scop_p scop, loop_p old_loop_father, - loop_iv_stack ivstack) +rename_variables (basic_block bb, htab_t map) { - basic_block bb = GBB_BB (gbb); gimple_stmt_iterator gsi; - for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi);) - { - gimple stmt = gsi_stmt (gsi); - - if (gimple_get_lhs (stmt) - && TREE_CODE (gimple_get_lhs (stmt)) == SSA_NAME - && get_old_iv_from_ssa_name (scop, old_loop_father, - gimple_get_lhs (stmt))) - gsi_remove (&gsi, false); - else - { - graphite_rename_ivs_stmt (stmt, gbb, scop, old_loop_father, ivstack); - gsi_next (&gsi); - } - } -} - -/* Move all the PHI nodes from block FROM to block TO. - OLD_LOOP_FATHER is the original loop that contained FROM. */ - -static void -move_phi_nodes (scop_p scop, loop_p old_loop_father, basic_block from, - basic_block to) -{ - gimple_stmt_iterator gsi; - - for (gsi = gsi_start_phis (from); !gsi_end_p (gsi);) - { - gimple phi = gsi_stmt (gsi); - tree op = gimple_phi_result (phi); - - if (get_old_iv_from_ssa_name (scop, old_loop_father, op) == NULL) - { - gimple new_phi = make_phi_node (op, 0); - add_phi_node_to_bb (new_phi, to); - } - remove_phi_node (&gsi, false); - } + for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi)) + rename_variables_in_stmt (gsi_stmt (gsi), map); } /* Remove condition from BB. */ @@ -3727,6 +3829,131 @@ get_true_edge_from_guard_bb (basic_block bb) return NULL; } +/* Returns the first successor edge of BB with EDGE_TRUE_VALUE flag cleared. */ + +static edge +get_false_edge_from_guard_bb (basic_block bb) +{ + edge e; + edge_iterator ei; + + FOR_EACH_EDGE (e, ei, bb->succs) + if (!(e->flags & EDGE_TRUE_VALUE)) + return e; + + gcc_unreachable (); + return NULL; +} + +/* Inserts in MAP a tuple (OLD_NAME, NEW_NAME) for the induction + variables of the loops around GBB in SCOP, i.e. GBB_LOOPS. + NEW_NAME is obtained from IVSTACK. IVSTACK has the same stack + ordering as GBB_LOOPS. */ + +static void +build_iv_mapping (loop_iv_stack ivstack, htab_t map, gbb_p gbb, scop_p scop) +{ + int i; + name_tree iv; + PTR *slot; + + for (i = 0; VEC_iterate (name_tree, SCOP_OLDIVS (scop), i, iv); i++) + { + struct rename_map_elt tmp; + + if (!flow_bb_inside_loop_p (iv->loop, GBB_BB (gbb))) + continue; + + tmp.old_name = iv->t; + slot = htab_find_slot (map, &tmp, INSERT); + + if (!*slot) + { + tree new_name = loop_iv_stack_get_iv (ivstack, + gbb_loop_index (gbb, iv->loop)); + *slot = new_rename_map_elt (iv->t, new_name); + } + } +} + +/* Register in MAP the tuple (old_name, new_name). */ + +static void +register_old_new_names (htab_t map, tree old_name, tree new_name) +{ + struct rename_map_elt tmp; + PTR *slot; + + tmp.old_name = old_name; + slot = htab_find_slot (map, &tmp, INSERT); + + if (!*slot) + *slot = new_rename_map_elt (old_name, new_name); +} + +/* Create a duplicate of the basic block BB. NOTE: This does not + preserve SSA form. */ + +static void +graphite_copy_stmts_from_block (basic_block bb, basic_block new_bb, htab_t map) +{ + gimple_stmt_iterator gsi, gsi_tgt; + + gsi_tgt = gsi_start_bb (new_bb); + for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) + { + def_operand_p def_p; + ssa_op_iter op_iter; + int region; + gimple stmt = gsi_stmt (gsi); + gimple copy; + + if (gimple_code (stmt) == GIMPLE_LABEL) + continue; + + /* Create a new copy of STMT and duplicate STMT's virtual + operands. */ + copy = gimple_copy (stmt); + gsi_insert_after (&gsi_tgt, copy, GSI_NEW_STMT); + mark_symbols_for_renaming (copy); + + region = lookup_stmt_eh_region (stmt); + if (region >= 0) + add_stmt_to_eh_region (copy, region); + gimple_duplicate_stmt_histograms (cfun, copy, cfun, stmt); + + /* Create new names for all the definitions created by COPY and + add replacement mappings for each new name. */ + FOR_EACH_SSA_DEF_OPERAND (def_p, copy, op_iter, SSA_OP_DEF) + { + tree old_name = DEF_FROM_PTR (def_p); + tree new_name = create_new_def_for (old_name, copy, def_p); + register_old_new_names (map, old_name, new_name); + } + } +} + +/* Copies BB and includes in the copied BB all the statements that can + be reached following the use-def chains from the memory accesses, + and returns the next edge following this new block. */ + +static edge +copy_bb_and_scalar_dependences (basic_block bb, scop_p scop, + loop_p context_loop, + edge next_e, htab_t map) +{ + basic_block new_bb = split_edge (next_e); + + next_e = single_succ_edge (new_bb); + graphite_copy_stmts_from_block (bb, new_bb, map); + remove_condition (new_bb); + rename_variables (new_bb, map); + remove_phi_nodes (new_bb); + expand_scalar_variables (new_bb, scop, context_loop, map); + + return next_e; +} + /* Translates a CLAST statement STMT to GCC representation. NEXT_E is the edge where new generated code should be attached. BB_EXIT is the last basic block that defines the scope of code generation. CONTEXT_LOOP is the @@ -3744,35 +3971,23 @@ translate_clast (scop_p scop, struct loop *context_loop, if (CLAST_STMT_IS_A (stmt, stmt_user)) { + htab_t map; CloogStatement *cs = ((struct clast_user_stmt *) stmt)->statement; graphite_bb_p gbb = (graphite_bb_p) cloog_statement_usr (cs); - basic_block bb = gbb->bb; - loop_p old_loop_father = bb->loop_father; - if (bb == ENTRY_BLOCK_PTR) + if (GBB_BB (gbb) == ENTRY_BLOCK_PTR) return next_e; - remove_condition (bb); - expand_scalar_variables (gbb, scop, old_loop_father, ivstack); - remove_all_edges (bb, next_e); - move_phi_nodes (scop, old_loop_father, bb, next_e->src); - redirect_edge_succ_nodup (next_e, bb); - - if (context_loop) - { - remove_bb_from_loops (bb); - add_bb_to_loop (bb, context_loop); - } - - set_immediate_dominator (CDI_DOMINATORS, next_e->dest, next_e->src); - mark_virtual_ops_in_bb (bb); - next_e = make_edge (bb, - context_loop ? context_loop->latch : EXIT_BLOCK_PTR, - EDGE_FALLTHRU); - loop_iv_stack_patch_for_consts (ivstack, - (struct clast_user_stmt *) stmt); - graphite_rename_ivs (gbb, scop, old_loop_father, ivstack); + map = htab_create (10, rename_map_elt_info, eq_rename_map_elts, free); + loop_iv_stack_patch_for_consts (ivstack, (struct clast_user_stmt *) stmt); + build_iv_mapping (ivstack, map, gbb, scop); + next_e = copy_bb_and_scalar_dependences (GBB_BB (gbb), scop, + context_loop, next_e, map); + htab_delete (map); loop_iv_stack_remove_constants (ivstack); + update_ssa (TODO_update_ssa); + recompute_all_dominators (); + graphite_verify (); return translate_clast (scop, context_loop, stmt->next, next_e, ivstack); } @@ -3790,7 +4005,9 @@ translate_clast (scop_p scop, struct loop *context_loop, set_immediate_dominator (CDI_DOMINATORS, next_e->dest, next_e->src); loop_iv_stack_pop (ivstack); - + last_e = single_succ_edge (split_edge (last_e)); + recompute_all_dominators (); + graphite_verify (); return translate_clast (scop, context_loop, stmt->next, last_e, ivstack); } @@ -3803,7 +4020,7 @@ translate_clast (scop_p scop, struct loop *context_loop, next_e = translate_clast (scop, context_loop, ((struct clast_guard *) stmt)->then, true_e, ivstack); - redirect_edge_succ_nodup (next_e, last_e->src); + graphite_verify (); return translate_clast (scop, context_loop, stmt->next, last_e, ivstack); } @@ -3812,6 +4029,7 @@ translate_clast (scop_p scop, struct loop *context_loop, next_e = translate_clast (scop, context_loop, ((struct clast_block *) stmt)->body, next_e, ivstack); + graphite_verify (); return translate_clast (scop, context_loop, stmt->next, next_e, ivstack); } @@ -4026,183 +4244,6 @@ find_transform (scop_p scop) return stmt; } -/* Return a vector of all the virtual phi nodes in the current - function. */ - -static VEC (gimple, heap) * -collect_virtual_phis (void) -{ - gimple_stmt_iterator si; - gimple_vec phis = VEC_alloc (gimple, heap, 3); - basic_block bb; - - FOR_EACH_BB (bb) - for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si)) - /* The phis we moved will have 0 arguments because the - original edges were removed. */ - if (gimple_phi_num_args (gsi_stmt (si)) == 0) - VEC_safe_push (gimple, heap, phis, gsi_stmt (si)); - - /* Deallocate if we did not find any. */ - if (VEC_length (gimple, phis) == 0) - { - VEC_free (gimple, heap, phis); - phis = NULL; - } - - return phis; -} - -/* Find a virtual definition for variable VAR in BB. */ - -static tree -find_vdef_for_var_in_bb (basic_block bb, tree var) -{ - gimple_stmt_iterator gsi; - gimple phi; - def_operand_p def_var; - vuse_vec_p vv; - ssa_op_iter op_iter; - - for (gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi)) - FOR_EACH_SSA_VDEF_OPERAND (def_var, vv, gsi_stmt (gsi), op_iter) - if (SSA_NAME_VAR (*def_var) == var) - return *def_var; - - for (gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi)) - FOR_EACH_SSA_DEF_OPERAND (def_var, gsi_stmt (gsi), op_iter, SSA_OP_DEF) - if (SSA_NAME_VAR (*def_var) == var) - return *def_var; - - for (gsi = gsi_start_phis (bb); !gsi_end_p(gsi); gsi_next (&gsi)) - { - phi = gsi_stmt (gsi); - if (SSA_NAME_VAR (PHI_RESULT (phi)) == var) - return PHI_RESULT (phi); - } - - return NULL; -} - -/* Recursive helper. */ - -static tree -find_vdef_for_var_1 (basic_block bb, struct pointer_set_t *visited, tree var) -{ - tree result = NULL; - edge_iterator ei; - edge pred_edge; - - if (pointer_set_contains (visited, bb)) - return NULL; - - pointer_set_insert (visited, bb); - result = find_vdef_for_var_in_bb (bb, var); - - if (!result) - FOR_EACH_EDGE (pred_edge, ei, bb->preds) - if (!result) - result = find_vdef_for_var_1 (pred_edge->src, visited, var); - - return result; -} - -/* Finds a virtual definition for variable VAR. */ - -static tree -find_vdef_for_var (basic_block bb, tree var) -{ - struct pointer_set_t *visited = pointer_set_create (); - tree def = find_vdef_for_var_1 (bb, visited, var); - - pointer_set_destroy (visited); - return def; -} - -/* Update the virtual phis after loop bodies are moved to new - loops. */ - -static void -patch_phis_for_virtual_defs (void) -{ - int i; - gimple phi; - VEC (gimple, heap) *virtual_phis = collect_virtual_phis (); - - for (i = 0; VEC_iterate (gimple, virtual_phis, i, phi); i++) - { - basic_block bb = gimple_bb (phi); - edge_iterator ei; - edge pred_edge; - gimple_stmt_iterator gsi; - gimple new_phi; - tree phi_result = PHI_RESULT (phi); - tree var = SSA_NAME_VAR (phi_result); - - new_phi = create_phi_node (phi_result, bb); - SSA_NAME_DEF_STMT (phi_result) = new_phi; - - FOR_EACH_EDGE (pred_edge, ei, bb->preds) - { - tree def = find_vdef_for_var (pred_edge->src, var); - - if (def) - add_phi_arg (new_phi, def, pred_edge); - else - add_phi_arg (new_phi, gimple_default_def (cfun, var), pred_edge); - } - - gsi = gsi_for_stmt (phi); - remove_phi_node (&gsi, false); - } - - VEC_free (gimple, heap, virtual_phis); -} - -/* Mark the original loops of SCOP for removal, replacing their header - field with NULL. */ - -static void -mark_old_loops (scop_p scop) -{ - int i; - struct loop *loop; - - for (i = 0; VEC_iterate (loop_p, SCOP_LOOP_NEST (scop), i, loop); i++) - { - loop->header = NULL; - loop->latch = NULL; - } -} - -/* Scan the loops and remove the ones that have been marked for - removal. */ - -static void -remove_dead_loops (void) -{ - struct loop *loop, *ploop; - loop_iterator li; - - FOR_EACH_LOOP (li, loop, LI_FROM_INNERMOST) - { - /* Remove only those loops that we marked to be removed with - mark_old_loops. */ - if (loop->header) - continue; - - while (loop->inner) - { - ploop = loop->inner; - flow_loop_tree_node_remove (ploop); - flow_loop_tree_node_add (loop_outer (loop), ploop); - } - - /* Remove the loop and free its data. */ - delete_loop (loop); - } -} - /* Returns true when it is possible to generate code for this STMT. For the moment we cannot generate code when Cloog decides to duplicate a statement, as we do not do a copy, but a move. @@ -4261,230 +4302,326 @@ can_generate_code (struct clast_stmt *stmt) return result; } -/* Skip any definition that is a phi node with a single phi def. */ +/* Remove from the CFG the REGION. */ -static tree -skip_phi_defs (tree ssa_name) +static inline void +remove_sese_region (sese region) { - tree result = ssa_name; - gimple def_stmt = SSA_NAME_DEF_STMT (ssa_name); + VEC (basic_block, heap) *bbs = NULL; + basic_block entry_bb = SESE_ENTRY (region)->dest; + basic_block exit_bb = SESE_EXIT (region)->dest; + basic_block bb; + int i; - if (gimple_code (def_stmt) == GIMPLE_PHI - && gimple_phi_num_args (def_stmt) == 1) - result = skip_phi_defs (gimple_phi_arg(def_stmt,0)->def); + VEC_safe_push (basic_block, heap, bbs, entry_bb); + gather_blocks_in_sese_region (entry_bb, exit_bb, &bbs); - return result; + for (i = 0; VEC_iterate (basic_block, bbs, i, bb); i++) + delete_basic_block (bb); + + VEC_free (basic_block, heap, bbs); } -/* Returns a VEC containing the phi-arg defs coming from SCOP_EXIT in - the destination block of SCOP_EXIT. */ +typedef struct ifsese { + sese region; + sese true_region; + sese false_region; +} *ifsese; -static VEC (tree, heap) * -collect_scop_exit_phi_args (edge scop_exit) +static inline edge +if_region_entry (ifsese if_region) { - VEC (tree, heap) *phi_args = VEC_alloc (tree, heap, 1); - gimple_stmt_iterator gsi; - - for (gsi = gsi_start_phis (scop_exit->dest); !gsi_end_p (gsi); gsi_next (&gsi)) - { - gimple phi = gsi_stmt (gsi); - tree phi_arg = skip_phi_defs(PHI_ARG_DEF_FROM_EDGE (phi, scop_exit)); - - VEC_safe_push (tree, heap, phi_args, phi_arg); - } + return SESE_ENTRY (if_region->region); +} - return phi_args; +static inline edge +if_region_exit (ifsese if_region) +{ + return SESE_EXIT (if_region->region); } -/* Patches (adds) PHI_ARGS to the phi nodes in SCOP_EXIT destination. */ +static inline basic_block +if_region_get_condition_block (ifsese if_region) +{ + return if_region_entry (if_region)->dest; +} -static void -patch_scop_exit_phi_args (edge scop_exit, - VEC (tree, heap) *phi_args) +static inline void +if_region_set_false_region (ifsese if_region, sese region) { - int i = 0; - gimple_stmt_iterator gsi; + basic_block condition = if_region_get_condition_block (if_region); + edge false_edge = get_false_edge_from_guard_bb (condition); + edge entry_region = SESE_ENTRY (region); + edge exit_region = SESE_EXIT (region); + basic_block before_region = entry_region->src; + basic_block last_in_region = exit_region->src; + void **slot = htab_find_slot_with_hash (current_loops->exits, exit_region, + htab_hash_pointer (exit_region), + NO_INSERT); + + entry_region->flags = false_edge->flags; + false_edge->flags = exit_region->flags; - for (gsi = gsi_start_phis (scop_exit->dest); !gsi_end_p (gsi); - gsi_next (&gsi), i++) + redirect_edge_pred (entry_region, condition); + redirect_edge_pred (exit_region, before_region); + redirect_edge_pred (false_edge, last_in_region); + + exit_region->flags = EDGE_FALLTHRU; + recompute_all_dominators (); + + SESE_EXIT (region) = single_succ_edge (false_edge->dest); + if_region->false_region = region; + + if (slot) { - tree def = VEC_index (tree, phi_args, i); - gimple phi = gsi_stmt (gsi); + struct loop_exit *loop_exit = GGC_CNEW (struct loop_exit); - gcc_assert (PHI_ARG_DEF_FROM_EDGE (phi, scop_exit) == NULL); + memcpy (loop_exit, *((struct loop_exit **) slot), sizeof (struct loop_exit)); + htab_clear_slot (current_loops->exits, slot); - add_phi_arg (phi, def, scop_exit); + slot = htab_find_slot_with_hash (current_loops->exits, false_edge, + htab_hash_pointer (false_edge), + INSERT); + loop_exit->e = false_edge; + *slot = loop_exit; + false_edge->src->loop_father->exits->next = loop_exit; } } -/* GIMPLE Loop Generator: generates loops from STMT in GIMPLE form for - the given SCOP. */ - -static void -gloog (scop_p scop, struct clast_stmt *stmt) +static ifsese +create_if_region_on_edge (edge entry, tree condition) { - edge new_scop_exit_edge = NULL; - basic_block scop_exit = SCOP_EXIT (scop); - VEC (tree, heap) *phi_args = - collect_scop_exit_phi_args (SESE_EXIT (SCOP_REGION (scop))); - VEC (iv_stack_entry_p, heap) *ivstack = - VEC_alloc (iv_stack_entry_p, heap, 10); - edge construction_edge = SESE_ENTRY (SCOP_REGION (scop)); - basic_block old_scop_exit_idom = get_immediate_dominator (CDI_DOMINATORS, - scop_exit); + edge e; + edge_iterator ei; + sese sese_region = GGC_NEW (struct sese); + sese true_region = GGC_NEW (struct sese); + sese false_region = GGC_NEW (struct sese); + ifsese if_region = GGC_NEW (struct ifsese); + edge exit = create_empty_if_region_on_edge (entry, condition); - if (!can_generate_code (stmt)) + if_region->region = sese_region; + if_region->region->entry = entry; + if_region->region->exit = exit; + + FOR_EACH_EDGE (e, ei, entry->dest->succs) { - cloog_clast_free (stmt); - return; + if (e->flags & EDGE_TRUE_VALUE) + { + true_region->entry = e; + true_region->exit = single_succ_edge (e->dest); + if_region->true_region = true_region; + } + else if (e->flags & EDGE_FALSE_VALUE) + { + false_region->entry = e; + false_region->exit = single_succ_edge (e->dest); + if_region->false_region = false_region; + } } - redirect_edge_succ_nodup (construction_edge, EXIT_BLOCK_PTR); - new_scop_exit_edge = translate_clast (scop, - construction_edge->src->loop_father, - stmt, construction_edge, &ivstack); - free_loop_iv_stack (&ivstack); - redirect_edge_succ (new_scop_exit_edge, scop_exit); + return if_region; +} - if (!old_scop_exit_idom - || !dominated_by_p (CDI_DOMINATORS, SCOP_ENTRY (scop), - old_scop_exit_idom) - || SCOP_ENTRY (scop) == old_scop_exit_idom) - set_immediate_dominator (CDI_DOMINATORS, - new_scop_exit_edge->dest, - new_scop_exit_edge->src); +/* Moves REGION in a condition expression: + | if (1) + | ; + | else + | REGION; +*/ - cloog_clast_free (stmt); +static ifsese +move_sese_in_condition (sese region) +{ + basic_block pred_block = split_edge (SESE_ENTRY (region)); + ifsese if_region = NULL; - if (new_scop_exit_edge->dest == EXIT_BLOCK_PTR) - new_scop_exit_edge->flags = 0; - - delete_unreachable_blocks (); - patch_phis_for_virtual_defs (); - patch_scop_exit_phi_args (new_scop_exit_edge, phi_args); - VEC_free (tree, heap, phi_args); - mark_old_loops (scop); - remove_dead_loops (); - rewrite_into_loop_closed_ssa (NULL, TODO_update_ssa); + SESE_ENTRY (region) = single_succ_edge (pred_block); + if_region = create_if_region_on_edge (single_pred_edge (pred_block), integer_one_node); + if_region_set_false_region (if_region, region); -#ifdef ENABLE_CHECKING - verify_loop_structure (); - verify_dominators (CDI_DOMINATORS); - verify_ssa (false); -#endif + return if_region; } -/* Returns the number of data references in SCOP. */ +/* Returns true when BB is in REGION. */ -static int -nb_data_refs_in_scop (scop_p scop) +static bool +bb_in_sese_p (basic_block bb, sese region) { - int i; - graphite_bb_p gbb; - int res = 0; + return (dominated_by_p (CDI_DOMINATORS, bb, SESE_ENTRY (region)->src) + && dominated_by_p (CDI_POST_DOMINATORS, bb, SESE_EXIT (region)->dest)); +} - for (i = 0; VEC_iterate (graphite_bb_p, SCOP_BBS (scop), i, gbb); i++) - res += VEC_length (data_reference_p, GBB_DATA_REFS (gbb)); +/* For USE in BB, if it is used outside of the REGION it is defined in, + mark it for rewrite. Record basic block BB where it is used + to USE_BLOCKS. Record the ssa name index to NEED_PHIS bitmap. */ - return res; +static void +sese_find_uses_to_rename_use (sese region, basic_block bb, tree use, + bitmap *use_blocks, bitmap need_phis) +{ + unsigned ver; + basic_block def_bb; + + if (TREE_CODE (use) != SSA_NAME) + return; + + ver = SSA_NAME_VERSION (use); + def_bb = gimple_bb (SSA_NAME_DEF_STMT (use)); + if (!def_bb + || !bb_in_sese_p (def_bb, region) + || bb_in_sese_p (bb, region)) + return; + + if (!use_blocks[ver]) + use_blocks[ver] = BITMAP_ALLOC (NULL); + bitmap_set_bit (use_blocks[ver], bb->index); + + bitmap_set_bit (need_phis, ver); } -/* Check if a graphite bb can be ignored in graphite. We ignore all - bbs, that only contain code, that will be eliminated later. +/* Marks names that are used in BB and outside of the loop they are + defined in for rewrite. Records the set of blocks in that the ssa + names are defined to USE_BLOCKS. Record the SSA names that will + need exit PHIs in NEED_PHIS. */ - TODO: - Move PHI nodes and scalar variables out of these bbs, that only - remain conditions and induction variables. */ +static void +sese_find_uses_to_rename_bb (sese region, basic_block bb, + bitmap *use_blocks, bitmap need_phis) +{ + gimple_stmt_iterator bsi; + edge e; + edge_iterator ei; + ssa_op_iter iter; + tree var; -static bool -gbb_can_be_ignored (graphite_bb_p gb) + FOR_EACH_EDGE (e, ei, bb->succs) + for (bsi = gsi_start_phis (e->dest); !gsi_end_p (bsi); gsi_next (&bsi)) + sese_find_uses_to_rename_use (region, bb, + PHI_ARG_DEF_FROM_EDGE (gsi_stmt (bsi), e), + use_blocks, need_phis); + + for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi)) + FOR_EACH_SSA_TREE_OPERAND (var, gsi_stmt (bsi), iter, SSA_OP_ALL_USES) + sese_find_uses_to_rename_use (region, bb, var, use_blocks, need_phis); +} + +/* Add exit phis for the USE on EXIT. */ + +static void +sese_add_exit_phis_edge (basic_block exit, tree use, edge false_e, edge true_e) { - gimple_stmt_iterator gsi; - scop_p scop = GBB_SCOP (gb); - loop_p loop = GBB_BB (gb)->loop_father; + gimple phi = create_phi_node (use, exit); - if (VEC_length (data_reference_p, GBB_DATA_REFS(gb))) - return false; + create_new_def_for (gimple_phi_result (phi), phi, + gimple_phi_result_ptr (phi)); + add_phi_arg (phi, use, false_e); + add_phi_arg (phi, integer_zero_node, true_e); +} - /* Check statements. */ - for (gsi = gsi_start_bb (GBB_BB (gb)); !gsi_end_p (gsi); gsi_next (&gsi)) - { - gimple stmt = gsi_stmt (gsi); - switch (gimple_code (stmt)) - { - /* Control flow expressions can be ignored, as they are - represented in the iteration domains and will be - regenerated by graphite. */ - case GIMPLE_COND: - case GIMPLE_GOTO: - case GIMPLE_SWITCH: - break; +/* Add phi nodes for VAR that is used in LIVEIN. Phi nodes are + inserted in block WHERE. */ - /* Scalar variables can be ignored, if we can regenerate - them later using their scalar evolution function. - XXX: Just a heuristic, that needs further investigation. */ - case GIMPLE_ASSIGN: - { - tree var = gimple_assign_lhs (stmt); - var = analyze_scalar_evolution (loop, var); - var = instantiate_scev (block_before_scop (scop), loop, var); +static void +sese_add_exit_phis_var (basic_block where, tree var, bitmap livein, + edge false_e, edge true_e) +{ + bitmap def; + basic_block def_bb = gimple_bb (SSA_NAME_DEF_STMT (var)); - if (TREE_CODE (var) == SCEV_NOT_KNOWN) - return false; + if (is_gimple_reg (var)) + bitmap_clear_bit (livein, def_bb->index); + else + bitmap_set_bit (livein, def_bb->index); - break; - } - /* Otherwise not ignoreable. */ - default: - return false; - } - } + def = BITMAP_ALLOC (NULL); + bitmap_set_bit (def, def_bb->index); + compute_global_livein (livein, def); + BITMAP_FREE (def); - return true; + sese_add_exit_phis_edge (where, var, false_e, true_e); } -/* Remove all ignoreable gbbs from SCOP. */ +/* Insert in the block WHERE phi nodes for variables defined in REGION + and used outside the REGION. */ static void -scop_remove_ignoreable_gbbs (scop_p scop) +rewrite_into_sese_closed_ssa (sese region, basic_block where, + edge false_e, edge true_e) { - graphite_bb_p gb; - int i; + unsigned i; + basic_block bb; + bitmap_iterator bi; + bitmap names_to_rename = BITMAP_ALLOC (NULL); + unsigned old_num_ssa_names = num_ssa_names; + bitmap *use_blocks = XCNEWVEC (bitmap, old_num_ssa_names); - int max_schedule = scop_max_loop_depth (scop) + 1; - lambda_vector last_schedule = lambda_vector_new (max_schedule); - lambda_vector_clear (last_schedule, max_schedule); + update_ssa (TODO_update_ssa); - /* Update schedules. */ - for (i = 0; VEC_iterate (graphite_bb_p, SCOP_BBS (scop), i, gb); i++) - { - int nb_loops = gbb_nb_loops (gb); + FOR_EACH_BB (bb) + sese_find_uses_to_rename_bb (region, bb, use_blocks, names_to_rename); - if (GBB_STATIC_SCHEDULE (gb) [nb_loops] == 0) - last_schedule [nb_loops] = 0; + EXECUTE_IF_SET_IN_BITMAP (names_to_rename, 0, i, bi) + sese_add_exit_phis_var (where, ssa_name (i), use_blocks[i], + false_e, true_e); - if (gbb_can_be_ignored (gb)) - { - /* Mark gbb for remove. */ - bitmap_clear_bit (SCOP_BBS_B (scop), gb->bb->index); - GBB_SCOP (gb) = NULL; - last_schedule [nb_loops]--; - } - else - lambda_vector_add (GBB_STATIC_SCHEDULE (gb), last_schedule, - GBB_STATIC_SCHEDULE (gb), nb_loops + 1); + update_ssa (TODO_update_ssa); + + for (i = 0; i < old_num_ssa_names; i++) + BITMAP_FREE (use_blocks[i]); + + free (use_blocks); + BITMAP_FREE (names_to_rename); +} + +/* GIMPLE Loop Generator: generates loops from STMT in GIMPLE form for + the given SCOP. */ + +static void +gloog (scop_p scop, struct clast_stmt *stmt) +{ + edge new_scop_exit_edge = NULL; + VEC (iv_stack_entry_p, heap) *ivstack = VEC_alloc (iv_stack_entry_p, heap, + 10); + loop_p context_loop; + ifsese if_region = NULL; + + if (!can_generate_code (stmt)) + { + cloog_clast_free (stmt); + return; } - /* Remove gbbs. */ - for (i = 0; VEC_iterate (graphite_bb_p, SCOP_BBS (scop), i, gb); i++) - if (GBB_SCOP (gb) == NULL) - { - VEC_unordered_remove (graphite_bb_p, SCOP_BBS (scop), i); - free_graphite_bb (gb); - /* XXX: Hackish? But working. */ - i--; - } + if_region = move_sese_in_condition (SCOP_REGION (scop)); + rewrite_into_sese_closed_ssa (SCOP_REGION (scop), + if_region->region->exit->src, + if_region->false_region->exit, + if_region->true_region->exit); + graphite_verify (); + context_loop = SESE_ENTRY (SCOP_REGION (scop))->src->loop_father; + new_scop_exit_edge = translate_clast (scop, context_loop, + stmt, if_region->true_region->entry, + &ivstack); + graphite_verify (); + cleanup_tree_cfg (); + recompute_all_dominators (); + graphite_verify (); + free_loop_iv_stack (&ivstack); + cloog_clast_free (stmt); +} - graphite_sort_gbbs (scop); +/* Returns the number of data references in SCOP. */ + +static int +nb_data_refs_in_scop (scop_p scop) +{ + int i; + graphite_bb_p gbb; + int res = 0; + + for (i = 0; VEC_iterate (graphite_bb_p, SCOP_BBS (scop), i, gbb); i++) + res += VEC_length (data_reference_p, GBB_DATA_REFS (gbb)); + + return res; } /* Move the loop at index LOOP and insert it before index NEW_LOOP_POS. @@ -4962,11 +5099,6 @@ graphite_trans_loop_block (VEC (graphite_bb_p, heap) *bbs, int loops) /* TODO: - Calculate the stride size automatically. */ int stride_size = 64; - /* It makes no sense to block a single loop. */ - for (i = 0; VEC_iterate (graphite_bb_p, bbs, i, gb); i++) - if (gbb_nb_loops (gb) < 2) - return false; - for (i = 0; VEC_iterate (graphite_bb_p, bbs, i, gb); i++) transform_done |= graphite_trans_bb_block (gb, stride_size, loops); @@ -5049,7 +5181,7 @@ graphite_trans_scop_block (scop_p scop) j++; /* Found perfect loop nest. */ - if (perfect && last_nb_loops - j > 0) + if (perfect && last_nb_loops - j >= 2) transform_done |= graphite_trans_loop_block (bbs, last_nb_loops - j); /* Check if we start with a new loop. @@ -5108,7 +5240,6 @@ graphite_apply_transformations (scop_p scop) /* Sort the list of bbs. Keep them always sorted. */ graphite_sort_gbbs (scop); - scop_remove_ignoreable_gbbs (scop); if (flag_loop_block) transform_done = graphite_trans_scop_block (scop); @@ -5157,13 +5288,15 @@ limit_scops (void) int j; loop_p loop; build_scop_bbs (scop); - build_scop_loop_nests (scop); + + if (!build_scop_loop_nests (scop)) + continue; for (j = 0; VEC_iterate (loop_p, SCOP_LOOP_NEST (scop), j, loop); j++) if (!loop_in_scop_p (loop_outer (loop), scop)) { sd_region open_scop; - open_scop.entry = loop_preheader_edge (loop)->dest; + open_scop.entry = loop->header; open_scop.exit = single_exit (loop)->dest; VEC_safe_push (sd_region, heap, tmp_scops, &open_scop); } @@ -5190,13 +5323,12 @@ graphite_transform_loops (void) return; current_scops = VEC_alloc (scop_p, heap, 3); - - calculate_dominance_info (CDI_DOMINATORS); - calculate_dominance_info (CDI_POST_DOMINATORS); + recompute_all_dominators (); if (dump_file && (dump_flags & TDF_DETAILS)) fprintf (dump_file, "Graphite loop transformations \n"); + initialize_original_copy_tables (); cloog_initialize (); build_scops (); limit_scops (); @@ -5208,9 +5340,12 @@ graphite_transform_loops (void) for (i = 0; VEC_iterate (scop_p, current_scops, i, scop); i++) { build_scop_bbs (scop); - build_scop_loop_nests (scop); + if (!build_scop_loop_nests (scop)) + continue; + build_scop_canonical_schedules (scop); build_bb_loops (scop); + build_scop_conditions (scop); find_scop_parameters (scop); build_scop_context (scop); @@ -5226,8 +5361,8 @@ graphite_transform_loops (void) if (!build_scop_iteration_domain (scop)) continue; - build_scop_conditions (scop); build_scop_data_accesses (scop); + build_scop_dynamic_schedules (scop); if (dump_file && (dump_flags & TDF_DETAILS)) { @@ -5249,6 +5384,7 @@ graphite_transform_loops (void) /* Cleanup. */ free_scops (current_scops); cloog_finalize (); + free_original_copy_tables (); } #else /* If Cloog is not available: #ifndef HAVE_cloog. */ |
