Delinearization of expressions that contain array size parameters

This allows us to delinerize code such as:

  A[][n]

  for (i
    for (j
      A[i][n-j-1] = ...

which would previously have been delinearize to an access A[i+1][-j-1].

To recover the correct access we apply the piecewise expression:

  { A[i][j] -> A[i-1][i+N]: i < 0; A[i][j] -> A[i][i]: i >= 0}

This approach generalizes to higher dimensions.

llvm-svn: 233566

8 files changed, 254 insertions, 14 deletions

diff --git a/polly/include/polly/ScopInfo.h b/polly/include/polly/ScopInfo.h
index 2b82065f4a0..606ef104220 100644
--- a/polly/include/polly/ScopInfo.h
+++ b/polly/include/polly/ScopInfo.h
@@ -223,6 +223,38 @@ private:
   /// @brief Get the new access function imported or set by a pass
   __isl_give isl_map *getNewAccessRelation() const;
 
+  /// @brief Fold the memory access to consider parameteric offsets
+  ///
+  /// To recover memory accesses with array size parameters in the subscript
+  /// expression we post-process the delinearization results.
+  ///
+  /// We would normally recover from an access A[exp0(i) * N + exp1(i)] into an
+  /// array A[][N] the 2D access A[exp0(i)][exp1(i)]. However, another valid
+  /// delinearization is A[exp0(i) - 1][exp1(i) + N] which - depending on the
+  /// range of exp1(i) - may be preferrable. Specifically, for cases where we
+  /// know exp1(i) is negative, we want to choose the latter expression.
+  ///
+  /// As we commonly do not have any information about the range of exp1(i),
+  /// we do not choose one of the two options, but instead create a piecewise
+  /// access function that adds the (-1, N) offsets as soon as exp1(i) becomes
+  /// negative. For a 2D array such an access function is created by applying
+  /// the piecewise map:
+  ///
+  /// [i,j] -> [i, j] :      j >= 0
+  /// [i,j] -> [i-1, j+N] :  j <  0
+  ///
+  /// We can generalize this mapping to arbitrary dimensions by applying this
+  /// piecewise mapping pairwise from the rightmost to the leftmost access
+  /// dimension. It would also be possible to cover a wider range by introducing
+  /// more cases and adding multiple of Ns to these cases. However, this has
+  /// not yet been necessary.
+  /// The introduction of different cases necessarily complicates the memory
+  /// access function, but cases that can be statically proven to not happen
+  /// will be eliminated later on.
+  __isl_give isl_map *foldAccess(const IRAccess &Access,
+                                 __isl_take isl_map *AccessRelation,
+                                 ScopStmt *Statement);
+
 public:
   /// @brief Create a memory access from an access in LLVM-IR.
   ///
diff --git a/polly/lib/Analysis/ScopInfo.cpp b/polly/lib/Analysis/ScopInfo.cpp
index d7b67d8be0b..e865a93e33e 100644
--- a/polly/lib/Analysis/ScopInfo.cpp
+++ b/polly/lib/Analysis/ScopInfo.cpp
@@ -567,6 +567,57 @@ void MemoryAccess::computeBoundsOnAccessRelation(unsigned ElementSize) {
   AccessRelation = isl_map_intersect_range(AccessRelation, AccessRange);
 }
 
+__isl_give isl_map *MemoryAccess::foldAccess(const IRAccess &Access,
+                                             __isl_take isl_map *AccessRelation,
+                                             ScopStmt *Statement) {
+  int Size = Access.Subscripts.size();
+
+  for (int i = Size - 2; i >= 0; --i) {
+    isl_space *Space;
+    isl_map *MapOne, *MapTwo;
+    isl_pw_aff *DimSize = SCEVAffinator::getPwAff(Statement, Access.Sizes[i]);
+
+    isl_space *SpaceSize = isl_pw_aff_get_space(DimSize);
+    isl_pw_aff_free(DimSize);
+    isl_id *ParamId = isl_space_get_dim_id(SpaceSize, isl_dim_param, 0);
+
+    Space = isl_map_get_space(AccessRelation);
+    Space = isl_space_map_from_set(isl_space_range(Space));
+    Space = isl_space_align_params(Space, SpaceSize);
+
+    int ParamLocation = isl_space_find_dim_by_id(Space, isl_dim_param, ParamId);
+    isl_id_free(ParamId);
+
+    MapOne = isl_map_universe(isl_space_copy(Space));
+    for (int j = 0; j < Size; ++j)
+      MapOne = isl_map_equate(MapOne, isl_dim_in, j, isl_dim_out, j);
+    MapOne = isl_map_lower_bound_si(MapOne, isl_dim_in, i + 1, 0);
+
+    MapTwo = isl_map_universe(isl_space_copy(Space));
+    for (int j = 0; j < Size; ++j)
+      if (j < i || j > i + 1)
+        MapTwo = isl_map_equate(MapTwo, isl_dim_in, j, isl_dim_out, j);
+
+    isl_local_space *LS = isl_local_space_from_space(Space);
+    isl_constraint *C;
+    C = isl_equality_alloc(isl_local_space_copy(LS));
+    C = isl_constraint_set_constant_si(C, -1);
+    C = isl_constraint_set_coefficient_si(C, isl_dim_in, i, 1);
+    C = isl_constraint_set_coefficient_si(C, isl_dim_out, i, -1);
+    MapTwo = isl_map_add_constraint(MapTwo, C);
+    C = isl_equality_alloc(LS);
+    C = isl_constraint_set_coefficient_si(C, isl_dim_in, i + 1, 1);
+    C = isl_constraint_set_coefficient_si(C, isl_dim_out, i + 1, -1);
+    C = isl_constraint_set_coefficient_si(C, isl_dim_param, ParamLocation, 1);
+    MapTwo = isl_map_add_constraint(MapTwo, C);
+    MapTwo = isl_map_upper_bound_si(MapTwo, isl_dim_in, i + 1, -1);
+
+    MapOne = isl_map_union(MapOne, MapTwo);
+    AccessRelation = isl_map_apply_range(AccessRelation, MapOne);
+  }
+  return AccessRelation;
+}
+
 MemoryAccess::MemoryAccess(const IRAccess &Access, Instruction *AccInst,
                            ScopStmt *Statement, const ScopArrayInfo *SAI)
     : AccType(getMemoryAccessType(Access)), Statement(Statement), Inst(AccInst),
@@ -616,6 +667,8 @@ MemoryAccess::MemoryAccess(const IRAccess &Access, Instruction *AccInst,
     AccessRelation = isl_map_flat_range_product(AccessRelation, SubscriptMap);
   }
 
+  AccessRelation = foldAccess(Access, AccessRelation, Statement);
+
   Space = Statement->getDomainSpace();
   AccessRelation = isl_map_set_tuple_id(
       AccessRelation, isl_dim_in, isl_space_get_tuple_id(Space, isl_dim_set));
diff --git a/polly/test/Isl/Ast/simple-run-time-condition.ll b/polly/test/Isl/Ast/simple-run-time-condition.ll
index 7138f58d9ce..7dc07f9610f 100644
--- a/polly/test/Isl/Ast/simple-run-time-condition.ll
+++ b/polly/test/Isl/Ast/simple-run-time-condition.ll
@@ -19,9 +19,9 @@ target triple = "x86_64-unknown-linux-gnu"
 ; cause any code to be executed are not generated.
 
 ; CHECK: if (
-; CHECK: ({{(q == 100 && o <= 0|o <= 0 && q == 100)}})
+; CHECK: (o >= 1 && q <= 0 && m + q >= 0)
 ; CHECK: ||
-; CHECK: ({{(q == 0 && o >= 1)|(o >= 1 && q == 0)}})
+; CHECK; (o <= 0 && m + q >= 100 && q <= 100)
 ; CHECK: )
 
 ; CHECK:     if (o >= 1) {
diff --git a/polly/test/ScopInfo/multidim_3d_parametric_array_static_loop_bounds.ll b/polly/test/ScopInfo/multidim_3d_parametric_array_static_loop_bounds.ll
index 0a02c71fc72..d692136a371 100644
--- a/polly/test/ScopInfo/multidim_3d_parametric_array_static_loop_bounds.ll
+++ b/polly/test/ScopInfo/multidim_3d_parametric_array_static_loop_bounds.ll
@@ -11,17 +11,17 @@ target triple = "x86_64-unknown-linux-gnu"
 ; }
 
 ; CHECK:   Assumed Context:
-; CHECK:   [m, o] -> {  : m >= 150 and o >= 200 }
-; CHECK:   p0: %m
-; CHECK:   p1: %o
+; CHECK:   [o, m] -> {  : m >= 150 and o >= 200 }
+; CHECK:   p0: %o
+; CHECK:   p1: %m
 ; CHECK:   Statements {
 ; CHECK:     Stmt_for_k
 ; CHECK:           Domain :=
-; CHECK:               [m, o] -> { Stmt_for_k[i0, i1, i2] : i0 >= 0 and i0 <= 99 and i1 >= 0 and i1 <= 149 and i2 >= 0 and i2 <= 199 };
+; CHECK:               [o, m] -> { Stmt_for_k[i0, i1, i2] : i0 >= 0 and i0 <= 99 and i1 >= 0 and i1 <= 149 and i2 >= 0 and i2 <= 199 };
 ; CHECK:           Scattering :=
-; CHECK:               [m, o] -> { Stmt_for_k[i0, i1, i2] -> [i0, i1, i2] };
+; CHECK:               [o, m] -> { Stmt_for_k[i0, i1, i2] -> [i0, i1, i2] };
 ; CHECK:           MustWriteAccess := [Reduction Type: NONE]
-; CHECK:               [m, o] -> { Stmt_for_k[i0, i1, i2] -> MemRef_A[i0, i1, i2] };
+; CHECK:               [o, m] -> { Stmt_for_k[i0, i1, i2] -> MemRef_A[i0, i1, i2] };
 
 define void @foo(i64 %n, i64 %m, i64 %o, double* %A) {
 entry:
diff --git a/polly/test/ScopInfo/multidim_ivs_and_parameteric_offsets_3d.ll b/polly/test/ScopInfo/multidim_ivs_and_parameteric_offsets_3d.ll
index 60284c3e793..200904c0c82 100644
--- a/polly/test/ScopInfo/multidim_ivs_and_parameteric_offsets_3d.ll
+++ b/polly/test/ScopInfo/multidim_ivs_and_parameteric_offsets_3d.ll
@@ -15,7 +15,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ;        (8 * %o)}<%for.j>,+,8}<%for.k>
 
 ; CHECK: Assumed Context:
-; CHECK: [n, m, o, p, q, r] -> { : q = 0 and r = 0 }
+; CHECK: [n, m, o, p, q, r] -> { : (q <= 0 and q >= 1 - m and r <= -1 and r >= 1 - o) or (r = 0 and q <= 0 and q >= -m) or (r = -o and q <= 1 and q >= 1 - m) }
 ;
 ; CHECK: p0: %n
 ; CHECK: p1: %m
@@ -30,7 +30,8 @@ target triple = "x86_64-unknown-linux-gnu"
 ; CHECK: Scattering
 ; CHECK:   [n, m, o, p, q, r] -> { Stmt_for_k[i0, i1, i2] -> [i0, i1, i2] };
 ; CHECK: MustWriteAccess
-; CHECK:   [n, m, o, p, q, r] -> { Stmt_for_k[i0, i1, i2] -> MemRef_A[p + i0, q + i1, r + i2] };
+; CHECK: [n, m, o, p, q, r] -> { Stmt_for_k[i0, i1, i2] -> MemRef_A[-1 + p + i0, -1 + m + q + i1, o + r + i2] : i1 <= -q and i2 <= -1 - r; Stmt_for_k[i0, i1, i2] -> MemRef_A[p + i0, -1 + q + i1, o + r + i2] : i1 >= 1 - q and i2 <= -1 - r; Stmt_for_k[i0, i1, i2] -> MemRef_A[-1 + p + i0, m + q + i1, r + i2] : i1 <= -1 - q and i2 >= -r; Stmt_for_k[i0, i1, i2] -> MemRef_A[p + i0, q + i1, r + i2] : i1 >= -q and i2 >= -r };
+
 
 define void @foo(i64 %n, i64 %m, i64 %o, double* %A, i64 %p, i64 %q, i64 %r) {
 entry:
diff --git a/polly/test/ScopInfo/multidim_only_ivs_3d_cast.ll b/polly/test/ScopInfo/multidim_only_ivs_3d_cast.ll
index 886b091f1e7..a726d703b3d 100644
--- a/polly/test/ScopInfo/multidim_only_ivs_3d_cast.ll
+++ b/polly/test/ScopInfo/multidim_only_ivs_3d_cast.ll
@@ -14,14 +14,14 @@
 
 ; CHECK: Assumed Context:
 ; CHECK:  [n, m, o, p_3, p_4] -> { :
-; CHECK-DAG: p_4 >= o
-; CHECK-DAG: p_3 >= m
+; CHECK-DAG: p_3 >= o
+; CHECK-DAG: p_4 >= m
 ; CHECK:  }
 ; CHECK: p0: %n
 ; CHECK: p1: %m
 ; CHECK: p2: %o
-; CHECK: p3: (zext i32 %m to i64)
-; CHECK: p4: (zext i32 %o to i64)
+; CHECK: p3: (zext i32 %o to i64)
+; CHECK: p4: (zext i32 %m to i64)
 ; CHECK-NOT: p5
 
 ; CHECK: Domain
diff --git a/polly/test/ScopInfo/multidim_param_in_subscript-2.ll b/polly/test/ScopInfo/multidim_param_in_subscript-2.ll
new file mode 100644
index 00000000000..303d9130ac7
--- /dev/null
+++ b/polly/test/ScopInfo/multidim_param_in_subscript-2.ll
@@ -0,0 +1,88 @@
+; RUN: opt %loadPolly -polly-scops -analyze < %s | FileCheck %s
+;
+;    void foo(long n, long m, float A[][n][m]) {
+;      for (long i = 0; i < 100; i++)
+;        for (long j = 0; j < n; j++)
+;          for (long k = 0; k < m; k++)
+;            A[i][j][k] += A[i][n - j - 1][m - k - 1];
+;    }
+;
+; Verify that the parameter in the subscript expression is correctly
+; recovered.
+;
+; CHECK: Assumed Context:
+; CHECK-NEXT: [n, m] -> {  :  }
+; CHECK: ReadAccess :=       [Reduction Type: NONE] [Scalar: 0]
+; CHECK-NEXT: [n, m] -> { Stmt_for_body6[i0, i1, i2] -> MemRef_A[i0, -1 + n - i1, -1 + m - i2] };
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @foo(i64 %n, i64 %m, float* %A) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc18, %entry
+  %i.0 = phi i64 [ 0, %entry ], [ %inc19, %for.inc18 ]
+  %exitcond = icmp ne i64 %i.0, 100
+  br i1 %exitcond, label %for.body, label %for.end20
+
+for.body:                                         ; preds = %for.cond
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %for.inc15, %for.body
+  %j.0 = phi i64 [ 0, %for.body ], [ %inc16, %for.inc15 ]
+  %cmp2 = icmp slt i64 %j.0, %n
+  br i1 %cmp2, label %for.body3, label %for.end17
+
+for.body3:                                        ; preds = %for.cond1
+  br label %for.cond4
+
+for.cond4:                                        ; preds = %for.inc, %for.body3
+  %k.0 = phi i64 [ 0, %for.body3 ], [ %inc, %for.inc ]
+  %cmp5 = icmp slt i64 %k.0, %m
+  br i1 %cmp5, label %for.body6, label %for.end
+
+for.body6:                                        ; preds = %for.cond4
+  %sub = sub nsw i64 %m, %k.0
+  %sub7 = add nsw i64 %sub, -1
+  %sub8 = sub nsw i64 %n, %j.0
+  %sub9 = add nsw i64 %sub8, -1
+  %tmp = mul nuw i64 %n, %m
+  %tmp1 = mul nsw i64 %i.0, %tmp
+  %tmp2 = mul nsw i64 %sub9, %m
+  %arrayidx.sum = add i64 %tmp1, %tmp2
+  %arrayidx10.sum = add i64 %arrayidx.sum, %sub7
+  %arrayidx11 = getelementptr inbounds float, float* %A, i64 %arrayidx10.sum
+  %tmp3 = load float, float* %arrayidx11, align 4
+  %tmp4 = mul nuw i64 %n, %m
+  %tmp5 = mul nsw i64 %i.0, %tmp4
+  %tmp6 = mul nsw i64 %j.0, %m
+  %arrayidx12.sum = add i64 %tmp5, %tmp6
+  %arrayidx13.sum = add i64 %arrayidx12.sum, %k.0
+  %arrayidx14 = getelementptr inbounds float, float* %A, i64 %arrayidx13.sum
+  %tmp7 = load float, float* %arrayidx14, align 4
+  %add = fadd float %tmp7, %tmp3
+  store float %add, float* %arrayidx14, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body6
+  %inc = add nuw nsw i64 %k.0, 1
+  br label %for.cond4
+
+for.end:                                          ; preds = %for.cond4
+  br label %for.inc15
+
+for.inc15:                                        ; preds = %for.end
+  %inc16 = add nuw nsw i64 %j.0, 1
+  br label %for.cond1
+
+for.end17:                                        ; preds = %for.cond1
+  br label %for.inc18
+
+for.inc18:                                        ; preds = %for.end17
+  %inc19 = add nuw nsw i64 %i.0, 1
+  br label %for.cond
+
+for.end20:                                        ; preds = %for.cond
+  ret void
+}
diff --git a/polly/test/ScopInfo/multidim_param_in_subscript.ll b/polly/test/ScopInfo/multidim_param_in_subscript.ll
new file mode 100644
index 00000000000..dfa925011ab
--- /dev/null
+++ b/polly/test/ScopInfo/multidim_param_in_subscript.ll
@@ -0,0 +1,66 @@
+; RUN: opt %loadPolly -polly-scops -analyze < %s | FileCheck %s
+;
+;
+;    void foo(long n, float A[][n]) {
+;      for (long i = 0; i < 100; i++)
+;        for (long j = 0; j < n; j++)
+;          A[i][j] += A[i][n - j - 1];
+;    }
+;
+; Verify that the parameter in the subscript expression is correctly
+; recovered.
+;
+; CHECK: Assumed Context:
+; CHECK-NEXT: [n] -> {  :  }
+;
+; CHECK: ReadAccess :=       [Reduction Type: NONE] [Scalar: 0]
+; CHECK-NEXT:   [n] -> { Stmt_for_body3[i0, i1] -> MemRef_A[i0, -1 + n - i1] };
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @foo(i64 %n, float* %A) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc8, %entry
+  %i.0 = phi i64 [ 0, %entry ], [ %inc9, %for.inc8 ]
+  %exitcond = icmp ne i64 %i.0, 100
+  br i1 %exitcond, label %for.body, label %for.end10
+
+for.body:                                         ; preds = %for.cond
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %for.inc, %for.body
+  %j.0 = phi i64 [ 0, %for.body ], [ %inc, %for.inc ]
+  %cmp2 = icmp slt i64 %j.0, %n
+  br i1 %cmp2, label %for.body3, label %for.end
+
+for.body3:                                        ; preds = %for.cond1
+  %sub = sub nsw i64 %n, %j.0
+  %sub4 = add nsw i64 %sub, -1
+  %tmp = mul nsw i64 %i.0, %n
+  %arrayidx.sum = add i64 %tmp, %sub4
+  %arrayidx5 = getelementptr inbounds float, float* %A, i64 %arrayidx.sum
+  %tmp1 = load float, float* %arrayidx5, align 4
+  %tmp2 = mul nsw i64 %i.0, %n
+  %arrayidx6.sum = add i64 %tmp2, %j.0
+  %arrayidx7 = getelementptr inbounds float, float* %A, i64 %arrayidx6.sum
+  %tmp3 = load float, float* %arrayidx7, align 4
+  %add = fadd float %tmp3, %tmp1
+  store float %add, float* %arrayidx7, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body3
+  %inc = add nuw nsw i64 %j.0, 1
+  br label %for.cond1
+
+for.end:                                          ; preds = %for.cond1
+  br label %for.inc8
+
+for.inc8:                                         ; preds = %for.end
+  %inc9 = add nuw nsw i64 %i.0, 1
+  br label %for.cond
+
+for.end10:                                        ; preds = %for.cond
+  ret void
+}