CachePruning: Allow limiting the number of files in the cache directory.

The default limit is 1000000 but it can be configured with a cache
policy. The motivation is that some filesystems (notably ext4) have
a limit on the number of files that can be contained in a directory
(separate from the inode limit).

Differential Revision: https://reviews.llvm.org/D40327

llvm-svn: 318857

4 files changed, 51 insertions, 20 deletions

diff --git a/clang/docs/ThinLTO.rst b/clang/docs/ThinLTO.rst
index 607121fd1e5..14c098b052d 100644
--- a/clang/docs/ThinLTO.rst
+++ b/clang/docs/ThinLTO.rst
@@ -173,6 +173,10 @@ Possible key-value pairs are:
   ``cache_size_bytes=1g`` on its own will cause both the 1GB and default 75%
   policies to be applied unless the default ``cache_size`` is overridden.
 
+- ``cache_size_files=X``:
+  Set the maximum number of files in the cache directory. Set to 0 to indicate
+  no limit. The default is 1000000 files.
+
 - ``prune_after=Xs``, ``prune_after=Xm``, ``prune_after=Xh``: Sets the
   expiration time for cache files to ``X`` seconds (or minutes, hours
   respectively).  When a file hasn't been accessed for ``prune_after`` seconds,
diff --git a/lld/test/ELF/lto/cache.ll b/lld/test/ELF/lto/cache.ll
index 2df8bafd411..5ab74f5c545 100644
--- a/lld/test/ELF/lto/cache.ll
+++ b/lld/test/ELF/lto/cache.ll
@@ -23,6 +23,14 @@
 ; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy cache_size_bytes=32k:prune_interval=0s -o %t3 %t2.o %t.o
 ; RUN: ls %t.cache | count 4
 
+; Setting max number of files to 0 should disable the limit, not delete everything.
+; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy prune_after=0s:cache_size=0%:cache_size_files=0:prune_interval=0s -o %t3 %t2.o %t.o
+; RUN: ls %t.cache | count 4
+
+; Delete everything except for the timestamp, "foo" and one cache file.
+; RUN: ld.lld --thinlto-cache-dir=%t.cache --thinlto-cache-policy prune_after=0s:cache_size=0%:cache_size_files=1:prune_interval=0s -o %t3 %t2.o %t.o
+; RUN: ls %t.cache | count 3
+
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/include/llvm/Support/CachePruning.h b/llvm/include/llvm/Support/CachePruning.h
index 46e34358573..c577e9b8b63 100644
--- a/llvm/include/llvm/Support/CachePruning.h
+++ b/llvm/include/llvm/Support/CachePruning.h
@@ -46,6 +46,15 @@ struct CachePruningPolicy {
   /// of available space on the disk will be reduced to the amount of available
   /// space. A value of 0 disables the absolute size-based pruning.
   uint64_t MaxSizeBytes = 0;
+
+  /// The maximum number of files in the cache directory. A value of 0 disables
+  /// the number of files based pruning.
+  ///
+  /// This defaults to 1000000 because with that many files there are
+  /// diminishing returns on the effectiveness of the cache, and some file
+  /// systems have a limit on how many files can be contained in a directory
+  /// (notably ext4, which is limited to around 6000000 files).
+  uint64_t MaxSizeFiles = 1000000;
 };
 
 /// Parse the given string as a cache pruning policy. Defaults are taken from a
diff --git a/llvm/lib/Support/CachePruning.cpp b/llvm/lib/Support/CachePruning.cpp
index 271cfc58c95..3e97c991f50 100644
--- a/llvm/lib/Support/CachePruning.cpp
+++ b/llvm/lib/Support/CachePruning.cpp
@@ -113,6 +113,10 @@ llvm::parseCachePruningPolicy(StringRef PolicyStr) {
         return make_error<StringError>("'" + Value + "' not an integer",
                                        inconvertibleErrorCode());
       Policy.MaxSizeBytes = Size * Mult;
+    } else if (Key == "cache_size_files") {
+      if (Value.getAsInteger(0, Policy.MaxSizeFiles))
+        return make_error<StringError>("'" + Value + "' not an integer",
+                                       inconvertibleErrorCode());
     } else {
       return make_error<StringError>("Unknown key: '" + Key + "'",
                                      inconvertibleErrorCode());
@@ -141,7 +145,7 @@ bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
 
   if (Policy.Expiration == seconds(0) &&
       Policy.MaxSizePercentageOfAvailableSpace == 0 &&
-      Policy.MaxSizeBytes == 0) {
+      Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
     DEBUG(dbgs() << "No pruning settings set, exit early\n");
     // Nothing will be pruned, early exit
     return false;
@@ -179,9 +183,6 @@ bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
     writeTimestampFile(TimestampFile);
   }
 
-  bool ShouldComputeSize =
-      (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0);
-
   // Keep track of space. Needs to be kept ordered by size for determinism.
   std::set<std::pair<uint64_t, std::string>> FileSizes;
   uint64_t TotalSize = 0;
@@ -211,7 +212,7 @@ bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
     // If the file hasn't been used recently enough, delete it
     const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
     auto FileAge = CurrentTime - FileAccessTime;
-    if (FileAge > Policy.Expiration) {
+    if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
       DEBUG(dbgs() << "Remove " << File->path() << " ("
                    << duration_cast<seconds>(FileAge).count() << "s old)\n");
       sys::fs::remove(File->path());
@@ -219,14 +220,32 @@ bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
     }
 
     // Leave it here for now, but add it to the list of size-based pruning.
-    if (!ShouldComputeSize)
-      continue;
     TotalSize += StatusOrErr->getSize();
     FileSizes.insert({StatusOrErr->getSize(), std::string(File->path())});
   }
 
+  auto FileAndSize = FileSizes.rbegin();
+  size_t NumFiles = FileSizes.size();
+
+  auto RemoveCacheFile = [&]() {
+    // Remove the file.
+    sys::fs::remove(FileAndSize->second);
+    // Update size
+    TotalSize -= FileAndSize->first;
+    NumFiles--;
+    DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
+                 << FileAndSize->first << "), new occupancy is " << TotalSize
+                 << "%\n");
+    ++FileAndSize;
+  };
+
+  // Prune for number of files.
+  if (Policy.MaxSizeFiles)
+    while (NumFiles > Policy.MaxSizeFiles)
+      RemoveCacheFile();
+
   // Prune for size now if needed
-  if (ShouldComputeSize) {
+  if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
     auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
     if (!ErrOrSpaceInfo) {
       report_fatal_error("Can't get available size");
@@ -246,18 +265,9 @@ bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
                  << "% target is: " << Policy.MaxSizePercentageOfAvailableSpace
                  << "%, " << Policy.MaxSizeBytes << " bytes\n");
 
-    auto FileAndSize = FileSizes.rbegin();
-    // Remove the oldest accessed files first, till we get below the threshold
-    while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend()) {
-      // Remove the file.
-      sys::fs::remove(FileAndSize->second);
-      // Update size
-      TotalSize -= FileAndSize->first;
-      DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
-                   << FileAndSize->first << "), new occupancy is " << TotalSize
-                   << "%\n");
-      ++FileAndSize;
-    }
+    // Remove the oldest accessed files first, till we get below the threshold.
+    while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend())
+      RemoveCacheFile();
   }
   return true;
 }