Avoid fragmenting the heap too much while reconstructing the index.

net/disk_cache/simple/simple_index_file.h

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef NET_DISK_CACHE_SIMPLE_SIMPLE_INDEX_FILE_H_
 #define NET_DISK_CACHE_SIMPLE_SIMPLE_INDEX_FILE_H_
 
 #include <string>
 #include <vector>
 
@@ skipping 79 matching lines @@
                            bool app_on_background);
 
   // Doom the entries specified in |entry_hashes|, calling |reply_callback|
   // with the result on the current thread when done.
   virtual void DoomEntrySet(scoped_ptr<std::vector<uint64> > entry_hashes,
                             const base::Callback<void(int)>& reply_callback);
 
  private:
   friend class WrappedSimpleIndexFile;
 
+  // Used for cache directory traversal.
+  typedef base::Callback<void (const std::string& file_name)> EntryFileCallback;
+
   // When loading the entries from disk, add this many extra hash buckets to
   // prevent reallocation on the IO thread when merging in new live entries.
   static const int kExtraSizeForMerge = 512;
 
   // Synchronous (IO performing) implementation of LoadIndexEntries.
   static void SyncLoadIndexEntries(base::Time cache_last_modified,
                                    const base::FilePath& cache_directory,
                                    const base::FilePath& index_file_path,
                                    SimpleIndexLoadResult* out_result);
 
   // Load the index file from disk returning an EntrySet. Upon failure, returns
   // NULL.
   static void SyncLoadFromDisk(const base::FilePath& index_filename,
                                SimpleIndexLoadResult* out_result);
 
   // Returns a scoped_ptr for a newly allocated Pickle containing the serialized
   // data to be written to a file.
   static scoped_ptr<Pickle> Serialize(
       const SimpleIndexFile::IndexMetadata& index_metadata,
       const SimpleIndex::EntrySet& entries);
 
   // Given the contents of an index file |data| of length |data_len|, returns
   // the corresponding EntrySet. Returns NULL on error.
   static void Deserialize(const char* data, int data_len,
                           SimpleIndexLoadResult* out_result);
 
+  // Implemented either in simple_index_file_posix.cc or
+  // simple_index_file_win.cc. base::FileEnumerator turned out to be very
+  // expensive in terms of memory usage therefore it's used only on non-POSIX
+  // environments for convenience (for now). Note that this function is
+  // synchronous although its signature could suggest otherwise.  Returns
+  // whether the traversal succeeded.
+  static bool TraverseCacheDirectory(
+      const std::string& cache_path,
+      const EntryFileCallback& entry_file_callback);
+
+  // Called for each cache directory traversal iteration. The provided
+  // non-cleared |buffer| can be used accross iterations to avoid doing too many
+  // heap allocations given that cache directories can be very large.
+  static void ProcessEntryFile(const std::string& cache_path,

[gavinp, 2013/08/20 20:13:45]

I suggest you want:

static void ProcessEntryFile(SimpleIndex::EntrySet* entries,
                                          uint64 entry_hash,
                                          const base::PlatformFileInfo&
entry_info);

Which can be for-sure-for-sure not allocating.

[Philippe, 2013/08/21 09:37:33]

I want to keep the hash extraction in this method to avoid duplicating the logic
on Windows and POSIX (unless I'm missing your point). Let me know if you are ok
with the new signature.

+                               std::string* buffer,

[gavinp, 2013/08/20 18:53:23]

I'm a bit confused at this buffer being a std::string, it's possible that the
std::string will reallocate on every call to std::string::append or
std::string::assign, isn't it?

[Philippe, 2013/08/21 09:37:33]

I replaced it with a vector that does now a single resize (and uses std::copy
with a non-back_inserter iterator).

+                               SimpleIndex::EntrySet* entries,
+                               const std::string& file_name);
+
   // Scan the index directory for entries, returning an EntrySet of all entries
   // found.
   static void SyncRestoreFromDisk(const base::FilePath& cache_directory,
                                   const base::FilePath& index_file_path,
                                   SimpleIndexLoadResult* out_result);
 
   // Determines if an index file is stale relative to the time of last
   // modification of the cache directory.
   static bool IsIndexFileStale(base::Time cache_last_modified,
                                const base::FilePath& index_file_path);
@@ skipping 11 matching lines @@
   static const char kIndexFileName[];
   static const char kTempIndexFileName[];
 
   DISALLOW_COPY_AND_ASSIGN(SimpleIndexFile);
 };
 
 
 }  // namespace disk_cache
 
 #endif  // NET_DISK_CACHE_SIMPLE_SIMPLE_INDEX_FILE_H_