[mlir] Refactor operation results to use a single use list for all results of the operation.

Summary: A new class is added, IRMultiObjectWithUseList, that allows for representing an IR use list that holds multiple sub values(used in this case for OpResults). This class provides all of the same functionality as the base IRObjectWithUseList, but for specific sub-values. This saves a word per operation result and is a necessary step in optimizing the layout of operation results. For now the use list is placed on the operation itself, so zero-result operations grow by a word. When the work for optimizing layout is finished, this can be moved back to being a trailing object based on memory/runtime benchmarking.

Reviewed By: jpienaar

Differential Revision: https://reviews.llvm.org/D71955

4 files changed, 292 insertions, 174 deletions

diff --git a/mlir/include/mlir/IR/Block.h b/mlir/include/mlir/IR/Block.h
index 934eed93c3b..c868148f95e 100644
--- a/mlir/include/mlir/IR/Block.h
+++ b/mlir/include/mlir/IR/Block.h
@@ -18,7 +18,7 @@
 
 namespace mlir {
 /// `Block` represents an ordered list of `Operation`s.
-class Block : public IRObjectWithUseList,
+class Block : public IRObjectWithUseList<BlockOperand>,
               public llvm::ilist_node_with_parent<Block, Region> {
 public:
   explicit Block() {}
diff --git a/mlir/include/mlir/IR/Operation.h b/mlir/include/mlir/IR/Operation.h
index 7586fc88337..4bc798834fc 100644
--- a/mlir/include/mlir/IR/Operation.h
+++ b/mlir/include/mlir/IR/Operation.h
@@ -25,7 +25,8 @@ namespace mlir {
 /// operations are organized into operation blocks represented by a 'Block'
 /// class.
 class Operation final
-    : public llvm::ilist_node_with_parent<Operation, Block>,
+    : public IRMultiObjectWithUseList<OpOperand>,
+      public llvm::ilist_node_with_parent<Operation, Block>,
       private llvm::TrailingObjects<Operation, OpResult, BlockOperand, Region,
                                     detail::OperandStorage> {
 public:
@@ -237,9 +238,6 @@ public:
   // Results
   //===--------------------------------------------------------------------===//
 
-  /// Return true if there are no users of any results of this operation.
-  bool use_empty();
-
   unsigned getNumResults() { return numResults; }
 
   Value getResult(unsigned idx) { return getOpResult(idx); }
@@ -639,32 +637,6 @@ inline raw_ostream &operator<<(raw_ostream &os, Operation &op) {
   return os;
 }
 
-/// This class implements use iterator for the Operation. This iterates over all
-/// uses of all results of an Operation.
-class UseIterator final
-    : public llvm::iterator_facade_base<UseIterator, std::forward_iterator_tag,
-                                        Operation *> {
-public:
-  /// Initialize UseIterator for op, specify end to return iterator to last use.
-  explicit UseIterator(Operation *op, bool end = false);
-
-  UseIterator &operator++();
-  Operation *operator->() { return use->getOwner(); }
-  Operation *operator*() { return use->getOwner(); }
-
-  bool operator==(const UseIterator &other) const;
-  bool operator!=(const UseIterator &other) const;
-
-private:
-  void skipOverResultsWithNoUsers();
-
-  /// The operation whose uses are being iterated over.
-  Operation *op;
-  /// The result of op who's uses are being iterated over.
-  Operation::result_iterator res;
-  /// The use of the result.
-  Value::use_iterator use;
-};
 } // end namespace mlir
 
 namespace llvm {
diff --git a/mlir/include/mlir/IR/UseDefLists.h b/mlir/include/mlir/IR/UseDefLists.h
index 222b50f8960..6b30f7488f7 100644
--- a/mlir/include/mlir/IR/UseDefLists.h
+++ b/mlir/include/mlir/IR/UseDefLists.h
@@ -20,65 +20,162 @@
 namespace mlir {
 
 class Block;
-class IROperand;
 class Operation;
 class Value;
 template <typename OperandType> class ValueUseIterator;
-template <typename OperandType> class ValueUserIterator;
+template <typename OperandType> class FilteredValueUseIterator;
+template <typename UseIteratorT, typename OperandType> class ValueUserIterator;
 
 //===----------------------------------------------------------------------===//
 // IRObjectWithUseList
 //===----------------------------------------------------------------------===//
 
-class IRObjectWithUseList {
+/// This class represents a single IR object that contains a use list.
+template <typename OperandType> class IRObjectWithUseList {
 public:
   ~IRObjectWithUseList() {
     assert(use_empty() && "Cannot destroy a value that still has uses!");
   }
 
-  /// Returns true if this value has no uses.
-  bool use_empty() const { return firstUse == nullptr; }
+  /// Drop all uses of this object from their respective owners.
+  void dropAllUses() {
+    while (!use_empty())
+      use_begin()->drop();
+  }
 
-  /// Returns true if this value has exactly one use.
-  inline bool hasOneUse() const;
+  /// Replace all uses of 'this' value with the new value, updating anything in
+  /// the IR that uses 'this' to use the other value instead.  When this returns
+  /// there are zero uses of 'this'.
+  void replaceAllUsesWith(typename OperandType::ValueType newValue) {
+    assert(this != OperandType::getUseList(newValue) &&
+           "cannot RAUW a value with itself");
+    while (!use_empty())
+      use_begin()->set(newValue);
+  }
 
-  using use_iterator = ValueUseIterator<IROperand>;
+  //===--------------------------------------------------------------------===//
+  // Uses
+  //===--------------------------------------------------------------------===//
+
+  using use_iterator = ValueUseIterator<OperandType>;
   using use_range = iterator_range<use_iterator>;
 
-  inline use_iterator use_begin() const;
-  inline use_iterator use_end() const;
+  use_iterator use_begin() const { return use_iterator(firstUse); }
+  use_iterator use_end() const { return use_iterator(nullptr); }
 
   /// Returns a range of all uses, which is useful for iterating over all uses.
-  inline use_range getUses() const;
+  use_range getUses() const { return {use_begin(), use_end()}; }
 
-  using user_iterator = ValueUserIterator<IROperand>;
-  using user_range = iterator_range<user_iterator>;
+  /// Returns true if this value has exactly one use.
+  bool hasOneUse() const {
+    return firstUse && firstUse->getNextOperandUsingThisValue() == nullptr;
+  }
 
-  inline user_iterator user_begin() const;
-  inline user_iterator user_end() const;
+  /// Returns true if this value has no uses.
+  bool use_empty() const { return firstUse == nullptr; }
 
-  /// Returns a range of all users.
-  inline user_range getUsers() const;
+  //===--------------------------------------------------------------------===//
+  // Users
+  //===--------------------------------------------------------------------===//
 
-  /// Replace all uses of 'this' value with the new value, updating anything in
-  /// the IR that uses 'this' to use the other value instead.  When this returns
-  /// there are zero uses of 'this'.
-  void replaceAllUsesWith(IRObjectWithUseList *newValue);
+  using user_iterator = ValueUserIterator<use_iterator, OperandType>;
+  using user_range = iterator_range<user_iterator>;
 
-  /// Drop all uses of this object from their respective owners.
-  void dropAllUses();
+  user_iterator user_begin() const { return user_iterator(use_begin()); }
+  user_iterator user_end() const { return user_iterator(use_end()); }
+
+  /// Returns a range of all users.
+  user_range getUsers() const { return {user_begin(), user_end()}; }
 
 protected:
   IRObjectWithUseList() {}
 
-  /// Return the first IROperand that is using this value, for use by custom
+  /// Return the first operand that is using this value, for use by custom
   /// use/def iterators.
-  IROperand *getFirstUse() { return firstUse; }
-  const IROperand *getFirstUse() const { return firstUse; }
+  OperandType *getFirstUse() const { return firstUse; }
 
 private:
-  friend class IROperand;
-  IROperand *firstUse = nullptr;
+  template <typename DerivedT, typename IRValueTy> friend class IROperand;
+  OperandType *firstUse = nullptr;
+};
+
+//===----------------------------------------------------------------------===//
+// IRMultiObjectWithUseList
+//===----------------------------------------------------------------------===//
+
+/// This class represents multiple IR objects with a single use list. This class
+/// provides wrapper functionality for manipulating the uses of a single object.
+template <typename OperandType>
+class IRMultiObjectWithUseList : public IRObjectWithUseList<OperandType> {
+public:
+  using BaseType = IRObjectWithUseList<OperandType>;
+  using ValueType = typename OperandType::ValueType;
+
+  /// Drop all uses of `value` from their respective owners.
+  void dropAllUses(ValueType value) {
+    assert(this == OperandType::getUseList(value) &&
+           "value not attached to this use list");
+    for (OperandType &use : llvm::make_early_inc_range(getUses(value)))
+      use.drop();
+  }
+  using BaseType::dropAllUses;
+
+  /// Replace all uses of `oldValue` with the new value, updating anything in
+  /// the IR that uses 'this' to use the other value instead.  When this returns
+  /// there are zero uses of 'this'.
+  void replaceAllUsesWith(ValueType oldValue, ValueType newValue) {
+    assert(this == OperandType::getUseList(oldValue) &&
+           "value not attached to this use list");
+    assert(this != OperandType::getUseList(newValue) &&
+           "cannot RAUW a value with itself");
+    for (OperandType &use : llvm::make_early_inc_range(getUses(oldValue)))
+      use.set(newValue);
+  }
+
+  //===--------------------------------------------------------------------===//
+  // Uses
+  //===--------------------------------------------------------------------===//
+
+  using filtered_use_iterator = FilteredValueUseIterator<OperandType>;
+  using filtered_use_range = iterator_range<filtered_use_iterator>;
+
+  filtered_use_iterator use_begin(ValueType value) const {
+    return filtered_use_iterator(this->getFirstUse(), value);
+  }
+  filtered_use_iterator use_end(ValueType) const { return use_end(); }
+  filtered_use_range getUses(ValueType value) const {
+    return {use_begin(value), use_end(value)};
+  }
+  bool hasOneUse(ValueType value) const {
+    return mlir::has_single_element(getUses(value));
+  }
+  bool use_empty(ValueType value) const {
+    return use_begin(value) == use_end(value);
+  }
+  using BaseType::getUses;
+  using BaseType::hasOneUse;
+  using BaseType::use_begin;
+  using BaseType::use_empty;
+  using BaseType::use_end;
+
+  //===--------------------------------------------------------------------===//
+  // Users
+  //===--------------------------------------------------------------------===//
+
+  using filtered_user_iterator =
+      ValueUserIterator<filtered_use_iterator, OperandType>;
+  using filtered_user_range = iterator_range<filtered_user_iterator>;
+
+  filtered_user_iterator user_begin(ValueType value) const {
+    return {use_begin(value)};
+  }
+  filtered_user_iterator user_end(ValueType value) const { return {use_end()}; }
+  filtered_user_range getUsers(ValueType value) const {
+    return {user_begin(value), user_end(value)};
+  }
+  using BaseType::getUsers;
+  using BaseType::user_begin;
+  using BaseType::user_end;
 };
 
 //===----------------------------------------------------------------------===//
@@ -86,19 +183,24 @@ private:
 //===----------------------------------------------------------------------===//
 
 /// A reference to a value, suitable for use as an operand of an operation.
-class IROperand {
+/// IRValueTy is the root type to use for values this tracks. Derived operand
+/// types are expected to provide the following:
+///  * static IRObjectWithUseList *getUseList(IRValueTy value);
+///    - Provide the use list that is attached to the given value.
+template <typename DerivedT, typename IRValueTy> class IROperand {
 public:
+  using ValueType = IRValueTy;
+
   IROperand(Operation *owner) : owner(owner) {}
-  IROperand(Operation *owner, IRObjectWithUseList *value)
-      : value(value), owner(owner) {
+  IROperand(Operation *owner, ValueType value) : value(value), owner(owner) {
     insertIntoCurrent();
   }
 
   /// Return the current value being used by this operand.
-  IRObjectWithUseList *get() const { return value; }
+  ValueType get() const { return value; }
 
   /// Set the current value being used by this operand.
-  void set(IRObjectWithUseList *newValue) {
+  void set(ValueType newValue) {
     // It isn't worth optimizing for the case of switching operands on a single
     // value.
     removeFromCurrent();
@@ -123,7 +225,7 @@ public:
   /// Return the next operand on the use-list of the value we are referring to.
   /// This should generally only be used by the internal implementation details
   /// of the SSA machinery.
-  IROperand *getNextOperandUsingThisValue() { return nextUse; }
+  DerivedT *getNextOperandUsingThisValue() { return nextUse; }
 
   /// We support a move constructor so IROperand's can be in vectors, but this
   /// shouldn't be used by general clients.
@@ -143,15 +245,15 @@ public:
   }
 
 private:
-  /// The value used as this operand.  This can be null when in a
-  /// "dropAllUses" state.
-  IRObjectWithUseList *value = nullptr;
+  /// The value used as this operand. This can be null when in a 'dropAllUses'
+  /// state.
+  ValueType value = {};
 
   /// The next operand in the use-chain.
-  IROperand *nextUse = nullptr;
+  DerivedT *nextUse = nullptr;
 
   /// This points to the previous link in the use-chain.
-  IROperand **back = nullptr;
+  DerivedT **back = nullptr;
 
   /// The operation owner of this operand.
   Operation *const owner;
@@ -169,11 +271,12 @@ private:
   }
 
   void insertIntoCurrent() {
-    back = &value->firstUse;
-    nextUse = value->firstUse;
+    auto *useList = DerivedT::getUseList(value);
+    back = &useList->firstUse;
+    nextUse = useList->firstUse;
     if (nextUse)
       nextUse->back = &nextUse;
-    value->firstUse = this;
+    useList->firstUse = static_cast<DerivedT *>(this);
   }
 };
 
@@ -182,15 +285,12 @@ private:
 //===----------------------------------------------------------------------===//
 
 /// Terminator operations can have Block operands to represent successors.
-class BlockOperand : public IROperand {
+class BlockOperand : public IROperand<BlockOperand, Block *> {
 public:
-  using IROperand::IROperand;
+  using IROperand<BlockOperand, Block *>::IROperand;
 
-  /// Return the current block being used by this operand.
-  Block *get();
-
-  /// Set the current value being used by this operand.
-  void set(Block *block);
+  /// Provide the use list that is attached to the given block.
+  static IRObjectWithUseList<BlockOperand> *getUseList(Block *value);
 
   /// Return which operand this is in the operand list of the User.
   unsigned getOperandNumber();
@@ -207,17 +307,35 @@ private:
 // OpOperand
 //===----------------------------------------------------------------------===//
 
+namespace detail {
+/// This class provides an opaque type erased wrapper around a `Value`.
+class OpaqueValue {
+public:
+  /// Implicit conversion from 'Value'.
+  OpaqueValue(Value value);
+  OpaqueValue(std::nullptr_t = nullptr) : impl(nullptr) {}
+  OpaqueValue(const OpaqueValue &) = default;
+  OpaqueValue &operator=(const OpaqueValue &) = default;
+  operator bool() const { return impl; }
+
+  /// Implicit conversion back to 'Value'.
+  operator Value() const;
+
+private:
+  void *impl;
+};
+} // namespace detail
+
 /// A reference to a value, suitable for use as an operand of an operation.
-class OpOperand : public IROperand {
+class OpOperand : public IROperand<OpOperand, detail::OpaqueValue> {
 public:
-  OpOperand(Operation *owner) : IROperand(owner) {}
-  OpOperand(Operation *owner, Value value);
+  using IROperand<OpOperand, detail::OpaqueValue>::IROperand;
 
-  /// Return the current value being used by this operand.
-  Value get();
+  /// Provide the use list that is attached to the given value.
+  static IRObjectWithUseList<OpOperand> *getUseList(Value value);
 
-  /// Set the current value being used by this operand.
-  void set(Value newValue);
+  /// Return the current value being used by this operand.
+  Value get() const;
 
   /// Return which operand this is in the operand list of the User.
   unsigned getOperandNumber();
@@ -227,67 +345,101 @@ public:
 // ValueUseIterator
 //===----------------------------------------------------------------------===//
 
-/// An iterator over all uses of a ValueBase.
-template <typename OperandType>
-class ValueUseIterator
-    : public std::iterator<std::forward_iterator_tag, OperandType> {
+namespace detail {
+/// A base iterator class that allows for iterating over the uses of a value.
+/// This is templated to allow for derived iterators to override specific
+/// iterator methods.
+template <typename DerivedT, typename OperandType>
+class ValueUseIteratorImpl
+    : public llvm::iterator_facade_base<DerivedT, std::forward_iterator_tag,
+                                        OperandType> {
 public:
-  ValueUseIterator() = default;
-  explicit ValueUseIterator(OperandType *current) : current(current) {}
-  OperandType *operator->() const { return current; }
-  OperandType &operator*() const { return *current; }
+  template <typename T>
+  ValueUseIteratorImpl(const ValueUseIteratorImpl<T, OperandType> &other)
+      : current(other.getOperand()) {}
+  ValueUseIteratorImpl(OperandType *current = nullptr) : current(current) {}
 
   Operation *getUser() const { return current->getOwner(); }
+  OperandType *getOperand() const { return current; }
+
+  OperandType &operator*() const { return *current; }
 
-  ValueUseIterator &operator++() {
+  using llvm::iterator_facade_base<DerivedT, std::forward_iterator_tag,
+                                   OperandType>::operator++;
+  ValueUseIteratorImpl &operator++() {
     assert(current && "incrementing past end()!");
     current = (OperandType *)current->getNextOperandUsingThisValue();
     return *this;
   }
 
-  ValueUseIterator operator++(int unused) {
-    ValueUseIterator copy = *this;
-    ++*this;
-    return copy;
+  bool operator==(const ValueUseIteratorImpl &rhs) const {
+    return current == rhs.current;
   }
 
-  friend bool operator==(ValueUseIterator lhs, ValueUseIterator rhs) {
-    return lhs.current == rhs.current;
-  }
+protected:
+  OperandType *current;
+};
 
-  friend bool operator!=(ValueUseIterator lhs, ValueUseIterator rhs) {
-    return !(lhs == rhs);
-  }
+} // end namespace detail
 
-private:
-  OperandType *current;
+/// An iterator over all of the uses of an IR object.
+template <typename OperandType>
+class ValueUseIterator
+    : public detail::ValueUseIteratorImpl<ValueUseIterator<OperandType>,
+                                          OperandType> {
+public:
+  using detail::ValueUseIteratorImpl<ValueUseIterator<OperandType>,
+                                     OperandType>::ValueUseIteratorImpl;
 };
 
-inline auto IRObjectWithUseList::use_begin() const -> use_iterator {
-  return use_iterator(firstUse);
-}
+/// This class represents an iterator of the uses of a IR object that optionally
+/// filters on a specific sub-value. This allows for filtering the uses of an
+/// IRMultiObjectWithUseList.
+template <typename OperandType>
+class FilteredValueUseIterator
+    : public detail::ValueUseIteratorImpl<FilteredValueUseIterator<OperandType>,
+                                          OperandType> {
+public:
+  using BaseT =
+      detail::ValueUseIteratorImpl<FilteredValueUseIterator<OperandType>,
+                                   OperandType>;
+
+  FilteredValueUseIterator() = default;
+  FilteredValueUseIterator(const ValueUseIterator<OperandType> &it)
+      : BaseT(it), filterVal(nullptr) {}
+  FilteredValueUseIterator(OperandType *current,
+                           typename OperandType::ValueType filterVal)
+      : BaseT(current), filterVal(filterVal) {
+    findNextValid();
+  }
 
-inline auto IRObjectWithUseList::use_end() const -> use_iterator {
-  return use_iterator(nullptr);
-}
+  using BaseT::operator++;
+  FilteredValueUseIterator<OperandType> &operator++() {
+    BaseT::operator++();
+    findNextValid();
+    return *this;
+  }
 
-inline auto IRObjectWithUseList::getUses() const -> use_range {
-  return {use_begin(), use_end()};
-}
+private:
+  void findNextValid() {
+    if (!filterVal)
+      return;
+    while (this->current && ((OperandType *)this->current)->get() != filterVal)
+      BaseT::operator++();
+  }
 
-/// Returns true if this value has exactly one use.
-inline bool IRObjectWithUseList::hasOneUse() const {
-  return firstUse && firstUse->getNextOperandUsingThisValue() == nullptr;
-}
+  /// An optional value to use to filter specific uses.
+  typename OperandType::ValueType filterVal;
+};
 
 //===----------------------------------------------------------------------===//
 // ValueUserIterator
 //===----------------------------------------------------------------------===//
 
 /// An iterator over all users of a ValueBase.
-template <typename OperandType>
+template <typename UseIteratorT, typename OperandType>
 class ValueUserIterator final
-    : public llvm::mapped_iterator<ValueUseIterator<OperandType>,
+    : public llvm::mapped_iterator<UseIteratorT,
                                    Operation *(*)(OperandType &)> {
   static Operation *unwrap(OperandType &value) { return value.getOwner(); }
 
@@ -296,24 +448,12 @@ public:
   using reference = Operation *;
 
   /// Initializes the result type iterator to the specified result iterator.
-  ValueUserIterator(ValueUseIterator<OperandType> it)
-      : llvm::mapped_iterator<ValueUseIterator<OperandType>,
-                              Operation *(*)(OperandType &)>(it, &unwrap) {}
+  ValueUserIterator(UseIteratorT it)
+      : llvm::mapped_iterator<UseIteratorT, Operation *(*)(OperandType &)>(
+            it, &unwrap) {}
   Operation *operator->() { return **this; }
 };
 
-inline auto IRObjectWithUseList::user_begin() const -> user_iterator {
-  return user_iterator(use_begin());
-}
-
-inline auto IRObjectWithUseList::user_end() const -> user_iterator {
-  return user_iterator(use_end());
-}
-
-inline auto IRObjectWithUseList::getUsers() const -> user_range {
-  return {user_begin(), user_end()};
-}
-
 } // namespace mlir
 
 #endif
diff --git a/mlir/include/mlir/IR/Value.h b/mlir/include/mlir/IR/Value.h
index 85e23b6b756..acbab60554a 100644
--- a/mlir/include/mlir/IR/Value.h
+++ b/mlir/include/mlir/IR/Value.h
@@ -27,7 +27,7 @@ class Value;
 
 namespace detail {
 /// The internal implementation of a Value.
-class ValueImpl : public IRObjectWithUseList {
+class ValueImpl {
 protected:
   /// This enumerates all of the SSA value kinds.
   enum class Kind {
@@ -46,7 +46,8 @@ private:
 };
 
 /// The internal implementation of a BlockArgument.
-class BlockArgumentImpl : public ValueImpl {
+class BlockArgumentImpl : public ValueImpl,
+                          public IRObjectWithUseList<OpOperand> {
   BlockArgumentImpl(Type type, Block *owner)
       : ValueImpl(Kind::BlockArgument, type), owner(owner) {}
 
@@ -132,13 +133,6 @@ public:
   /// place.
   void setType(Type newType) { impl->typeAndKind.setPointer(newType); }
 
-  /// Replace all uses of 'this' value with the new value, updating anything in
-  /// the IR that uses 'this' to use the other value instead.  When this returns
-  /// there are zero uses of 'this'.
-  void replaceAllUsesWith(Value newValue) const {
-    impl->replaceAllUsesWith(newValue.impl);
-  }
-
   /// If this value is the result of an operation, return the operation that
   /// defines it.
   Operation *getDefiningOp() const;
@@ -150,32 +144,52 @@ public:
   /// Return the Region in which this Value is defined.
   Region *getParentRegion();
 
-  using use_iterator = ValueUseIterator<OpOperand>;
+  //===--------------------------------------------------------------------===//
+  // UseLists
+  //===--------------------------------------------------------------------===//
+
+  /// Provide the use list that is attached to this value.
+  IRObjectWithUseList<OpOperand> *getUseList() const;
+
+  /// Drop all uses of this object from their respective owners.
+  void dropAllUses() const;
+
+  /// Replace all uses of 'this' value with the new value, updating anything in
+  /// the IR that uses 'this' to use the other value instead.  When this returns
+  /// there are zero uses of 'this'.
+  void replaceAllUsesWith(Value newValue) const;
+
+  //===--------------------------------------------------------------------===//
+  // Uses
+
+  /// This class implements an iterator over the uses of a value.
+  using use_iterator = FilteredValueUseIterator<OpOperand>;
   using use_range = iterator_range<use_iterator>;
 
-  inline use_iterator use_begin();
-  inline use_iterator use_end();
+  use_iterator use_begin() const;
+  use_iterator use_end() const { return use_iterator(); }
 
   /// Returns a range of all uses, which is useful for iterating over all uses.
-  inline use_range getUses();
+  use_range getUses() const { return {use_begin(), use_end()}; }
 
-  using user_iterator = ValueUserIterator<IROperand>;
-  using user_range = iterator_range<user_iterator>;
+  /// Returns true if this value has exactly one use.
+  bool hasOneUse() const;
 
-  user_iterator user_begin() const { return impl->user_begin(); }
-  user_iterator user_end() const { return impl->user_end(); }
+  /// Returns true if this value has no uses.
+  bool use_empty() const;
 
-  /// Returns a range of all users.
-  user_range getUsers() const { return impl->getUsers(); }
+  //===--------------------------------------------------------------------===//
+  // Users
 
-  /// Returns true if this value has no uses.
-  bool use_empty() const { return impl->use_empty(); }
+  using user_iterator = ValueUserIterator<use_iterator, OpOperand>;
+  using user_range = iterator_range<user_iterator>;
 
-  /// Returns true if this value has exactly one use.
-  bool hasOneUse() const { return impl->hasOneUse(); }
+  user_iterator user_begin() const { return use_begin(); }
+  user_iterator user_end() const { return use_end(); }
+  user_range getUsers() const { return {user_begin(), user_end()}; }
 
-  /// Drop all uses of this object from their respective owners.
-  void dropAllUses() const { impl->dropAllUses(); }
+  //===--------------------------------------------------------------------===//
+  // Utilities
 
   void print(raw_ostream &os);
   void dump();
@@ -201,17 +215,6 @@ inline raw_ostream &operator<<(raw_ostream &os, Value value) {
   return os;
 }
 
-// Utility functions for iterating through Value uses.
-inline auto Value::use_begin() -> use_iterator {
-  return use_iterator((OpOperand *)impl->getFirstUse());
-}
-
-inline auto Value::use_end() -> use_iterator { return use_iterator(nullptr); }
-
-inline auto Value::getUses() -> iterator_range<use_iterator> {
-  return {use_begin(), use_end()};
-}
-
 /// Block arguments are values.
 class BlockArgument : public Value {
 public:
@@ -248,6 +251,9 @@ private:
 
   /// Allow access to `create` and `destroy`.
   friend Block;
+
+  /// Allow access to 'getImpl'.
+  friend Value;
 };
 
 /// This is a value defined by a result of an operation.