Support: Have directory_iterator::status() return FindFirstFileEx/FindNextFile results on Windows.

This allows clients to avoid an unnecessary fs::status() call on each
directory entry. Because the information returned by FindFirstFileEx
is a subset of the information returned by a regular status() call,
I needed to extract a base class from file_status that contains only
that information.

On my machine, this reduces the time required to enumerate a ThinLTO
cache directory containing 520k files from almost 4 minutes to less
than 2 seconds.

Differential Revision: https://reviews.llvm.org/D38716

llvm-svn: 315378

4 files changed, 69 insertions, 53 deletions

diff --git a/llvm/lib/Support/CachePruning.cpp b/llvm/lib/Support/CachePruning.cpp
index 60d0964f276..5a9580cf440 100644
--- a/llvm/lib/Support/CachePruning.cpp
+++ b/llvm/lib/Support/CachePruning.cpp
@@ -182,19 +182,9 @@ bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
   bool ShouldComputeSize =
       (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0);
 
-  // Keep track of space
+  // Keep track of space. Needs to be kept ordered by size for determinism.
   std::set<std::pair<uint64_t, std::string>> FileSizes;
   uint64_t TotalSize = 0;
-  // Helper to add a path to the set of files to consider for size-based
-  // pruning, sorted by size.
-  auto AddToFileListForSizePruning =
-      [&](StringRef Path) {
-        if (!ShouldComputeSize)
-          return;
-        TotalSize += FileStatus.getSize();
-        FileSizes.insert(
-            std::make_pair(FileStatus.getSize(), std::string(Path)));
-      };
 
   // Walk the entire directory cache, looking for unused files.
   std::error_code EC;
@@ -212,13 +202,14 @@ bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
 
     // Look at this file. If we can't stat it, there's nothing interesting
     // there.
-    if (sys::fs::status(File->path(), FileStatus)) {
+    ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();
+    if (!StatusOrErr) {
       DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
       continue;
     }
 
     // If the file hasn't been used recently enough, delete it
-    const auto FileAccessTime = FileStatus.getLastAccessedTime();
+    const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
     auto FileAge = CurrentTime - FileAccessTime;
     if (FileAge > Policy.Expiration) {
       DEBUG(dbgs() << "Remove " << File->path() << " ("
@@ -228,7 +219,10 @@ bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
     }
 
     // Leave it here for now, but add it to the list of size-based pruning.
-    AddToFileListForSizePruning(File->path());
+    if (!ShouldComputeSize)
+      continue;
+    TotalSize += StatusOrErr->getSize();
+    FileSizes.insert({StatusOrErr->getSize(), std::string(File->path())});
   }
 
   // Prune for size now if needed
diff --git a/llvm/lib/Support/Path.cpp b/llvm/lib/Support/Path.cpp
index f30e8a8b0cb..9692acb5283 100644
--- a/llvm/lib/Support/Path.cpp
+++ b/llvm/lib/Support/Path.cpp
@@ -952,11 +952,11 @@ ErrorOr<MD5::MD5Result> md5_contents(const Twine &Path) {
   return Result;
 }
 
-bool exists(file_status status) {
+bool exists(const basic_file_status &status) {
   return status_known(status) && status.type() != file_type::file_not_found;
 }
 
-bool status_known(file_status s) {
+bool status_known(const basic_file_status &s) {
   return s.type() != file_type::status_error;
 }
 
@@ -967,7 +967,7 @@ file_type get_file_type(const Twine &Path, bool Follow) {
   return st.type();
 }
 
-bool is_directory(file_status status) {
+bool is_directory(const basic_file_status &status) {
   return status.type() == file_type::directory_file;
 }
 
@@ -979,7 +979,7 @@ std::error_code is_directory(const Twine &path, bool &result) {
   return std::error_code();
 }
 
-bool is_regular_file(file_status status) {
+bool is_regular_file(const basic_file_status &status) {
   return status.type() == file_type::regular_file;
 }
 
@@ -991,7 +991,7 @@ std::error_code is_regular_file(const Twine &path, bool &result) {
   return std::error_code();
 }
 
-bool is_symlink_file(file_status status) {
+bool is_symlink_file(const basic_file_status &status) {
   return status.type() == file_type::symlink_file;
 }
 
@@ -1003,7 +1003,7 @@ std::error_code is_symlink_file(const Twine &path, bool &result) {
   return std::error_code();
 }
 
-bool is_other(file_status status) {
+bool is_other(const basic_file_status &status) {
   return exists(status) &&
          !is_regular_file(status) &&
          !is_directory(status);
@@ -1017,17 +1017,14 @@ std::error_code is_other(const Twine &Path, bool &Result) {
   return std::error_code();
 }
 
-void directory_entry::replace_filename(const Twine &filename, file_status st) {
+void directory_entry::replace_filename(const Twine &filename,
+                                       basic_file_status st) {
   SmallString<128> path = path::parent_path(Path);
   path::append(path, filename);
   Path = path.str();
   Status = st;
 }
 
-std::error_code directory_entry::status(file_status &result) const {
-  return fs::status(Path, result, FollowSymlinks);
-}
-
 ErrorOr<perms> getPermissions(const Twine &Path) {
   file_status Status;
   if (std::error_code EC = status(Path, Status))
diff --git a/llvm/lib/Support/Unix/Path.inc b/llvm/lib/Support/Unix/Path.inc
index d0bb6a4fffb..781a911ed57 100644
--- a/llvm/lib/Support/Unix/Path.inc
+++ b/llvm/lib/Support/Unix/Path.inc
@@ -217,11 +217,11 @@ std::string getMainExecutable(const char *argv0, void *MainAddr) {
   return "";
 }
 
-TimePoint<> file_status::getLastAccessedTime() const {
+TimePoint<> basic_file_status::getLastAccessedTime() const {
   return toTimePoint(fs_st_atime);
 }
 
-TimePoint<> file_status::getLastModificationTime() const {
+TimePoint<> basic_file_status::getLastModificationTime() const {
   return toTimePoint(fs_st_mtime);
 }
 
@@ -713,6 +713,13 @@ std::error_code detail::directory_iterator_increment(detail::DirIterState &it) {
   return std::error_code();
 }
 
+ErrorOr<basic_file_status> directory_entry::status() const {
+  file_status s;
+  if (auto EC = fs::status(Path, s, FollowSymlinks))
+    return EC;
+  return s;
+}
+
 #if !defined(F_GETPATH)
 static bool hasProcSelfFD() {
   // If we have a /proc filesystem mounted, we can quickly establish the
@@ -809,12 +816,11 @@ static std::error_code remove_directories_impl(const T &Entry,
   directory_iterator End;
   while (Begin != End) {
     auto &Item = *Begin;
-    file_status st;
-    EC = Item.status(st);
-    if (EC && !IgnoreErrors)
-      return EC;
+    ErrorOr<basic_file_status> st = Item.status();
+    if (!st && !IgnoreErrors)
+      return st.getError();
 
-    if (is_directory(st)) {
+    if (is_directory(*st)) {
       EC = remove_directories_impl(Item, IgnoreErrors);
       if (EC && !IgnoreErrors)
         return EC;
diff --git a/llvm/lib/Support/Windows/Path.inc b/llvm/lib/Support/Windows/Path.inc
index a81cd58cf4b..1ab354bf094 100644
--- a/llvm/lib/Support/Windows/Path.inc
+++ b/llvm/lib/Support/Windows/Path.inc
@@ -168,14 +168,14 @@ ErrorOr<space_info> disk_space(const Twine &Path) {
   return SpaceInfo;
 }
 
-TimePoint<> file_status::getLastAccessedTime() const {
+TimePoint<> basic_file_status::getLastAccessedTime() const {
   FILETIME Time;
   Time.dwLowDateTime = LastAccessedTimeLow;
   Time.dwHighDateTime = LastAccessedTimeHigh;
   return toTimePoint(Time);
 }
 
-TimePoint<> file_status::getLastModificationTime() const {
+TimePoint<> basic_file_status::getLastModificationTime() const {
   FILETIME Time;
   Time.dwLowDateTime = LastWriteTimeLow;
   Time.dwHighDateTime = LastWriteTimeHigh;
@@ -569,6 +569,15 @@ static bool isReservedName(StringRef path) {
   return false;
 }
 
+static file_type file_type_from_attrs(DWORD Attrs) {
+  return (Attrs & FILE_ATTRIBUTE_DIRECTORY) ? file_type::directory_file
+                                            : file_type::regular_file;
+}
+
+static perms perms_from_attrs(DWORD Attrs) {
+  return (Attrs & FILE_ATTRIBUTE_READONLY) ? (all_read | all_exe) : all_all;
+}
+
 static std::error_code getStatus(HANDLE FileHandle, file_status &Result) {
   if (FileHandle == INVALID_HANDLE_VALUE)
     goto handle_status_error;
@@ -597,22 +606,14 @@ static std::error_code getStatus(HANDLE FileHandle, file_status &Result) {
   if (!::GetFileInformationByHandle(FileHandle, &Info))
     goto handle_status_error;
 
-  {
-    file_type Type = (Info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
-                         ? file_type::directory_file
-                         : file_type::regular_file;
-    perms Permissions = (Info.dwFileAttributes & FILE_ATTRIBUTE_READONLY)
-                            ? (all_read | all_exe)
-                            : all_all;
-    Result = file_status(
-        Type, Permissions, Info.nNumberOfLinks,
-        Info.ftLastAccessTime.dwHighDateTime,
-        Info.ftLastAccessTime.dwLowDateTime,
-        Info.ftLastWriteTime.dwHighDateTime, Info.ftLastWriteTime.dwLowDateTime,
-        Info.dwVolumeSerialNumber, Info.nFileSizeHigh, Info.nFileSizeLow,
-        Info.nFileIndexHigh, Info.nFileIndexLow);
-    return std::error_code();
-  }
+  Result = file_status(
+      file_type_from_attrs(Info.dwFileAttributes),
+      perms_from_attrs(Info.dwFileAttributes), Info.nNumberOfLinks,
+      Info.ftLastAccessTime.dwHighDateTime, Info.ftLastAccessTime.dwLowDateTime,
+      Info.ftLastWriteTime.dwHighDateTime, Info.ftLastWriteTime.dwLowDateTime,
+      Info.dwVolumeSerialNumber, Info.nFileSizeHigh, Info.nFileSizeLow,
+      Info.nFileIndexHigh, Info.nFileIndexLow);
+  return std::error_code();
 
 handle_status_error:
   DWORD LastError = ::GetLastError();
@@ -798,6 +799,16 @@ int mapped_file_region::alignment() {
   return SysInfo.dwAllocationGranularity;
 }
 
+static basic_file_status status_from_find_data(WIN32_FIND_DATA *FindData) {
+  return basic_file_status(file_type_from_attrs(FindData->dwFileAttributes),
+                           perms_from_attrs(FindData->dwFileAttributes),
+                           FindData->ftLastAccessTime.dwHighDateTime,
+                           FindData->ftLastAccessTime.dwLowDateTime,
+                           FindData->ftLastWriteTime.dwHighDateTime,
+                           FindData->ftLastWriteTime.dwLowDateTime,
+                           FindData->nFileSizeHigh, FindData->nFileSizeLow);
+}
+
 std::error_code detail::directory_iterator_construct(detail::DirIterState &it,
                                                      StringRef path,
                                                      bool follow_symlinks) {
@@ -818,7 +829,9 @@ std::error_code detail::directory_iterator_construct(detail::DirIterState &it,
 
   //  Get the first directory entry.
   WIN32_FIND_DATAW FirstFind;
-  ScopedFindHandle FindHandle(::FindFirstFileW(c_str(path_utf16), &FirstFind));
+  ScopedFindHandle FindHandle(::FindFirstFileExW(
+      c_str(path_utf16), FindExInfoBasic, &FirstFind, FindExSearchNameMatch,
+      NULL, FIND_FIRST_EX_LARGE_FETCH));
   if (!FindHandle)
     return mapWindowsError(::GetLastError());
 
@@ -845,7 +858,8 @@ std::error_code detail::directory_iterator_construct(detail::DirIterState &it,
   it.IterationHandle = intptr_t(FindHandle.take());
   SmallString<128> directory_entry_path(path);
   path::append(directory_entry_path, directory_entry_name_utf8);
-  it.CurrentEntry = directory_entry(directory_entry_path, follow_symlinks);
+  it.CurrentEntry = directory_entry(directory_entry_path, follow_symlinks,
+                                    status_from_find_data(&FirstFind));
 
   return std::error_code();
 }
@@ -881,10 +895,15 @@ std::error_code detail::directory_iterator_increment(detail::DirIterState &it) {
                       directory_entry_path_utf8))
     return ec;
 
-  it.CurrentEntry.replace_filename(Twine(directory_entry_path_utf8));
+  it.CurrentEntry.replace_filename(Twine(directory_entry_path_utf8),
+                                   status_from_find_data(&FindData));
   return std::error_code();
 }
 
+ErrorOr<basic_file_status> directory_entry::status() const {
+  return Status;
+}
+
 static std::error_code realPathFromHandle(HANDLE H,
                                           SmallVectorImpl<char> &RealPath) {
   RealPath.clear();