Add a PartialPreReadImage function to file_util (on Windows).

base/file_util_win.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/file_util.h"
 
 #include <windows.h>
 #include <propvarutil.h>
 #include <psapi.h>
 #include <shellapi.h>
 #include <shlobj.h>
 #include <time.h>
 
+#include <algorithm>
 #include <limits>
 #include <string>
 
 #include "base/file_path.h"
 #include "base/logging.h"
+#include "base/memory/scoped_ptr.h"
 #include "base/metrics/histogram.h"
 #include "base/string_number_conversions.h"
 #include "base/string_util.h"
 #include "base/threading/thread_restrictions.h"
 #include "base/time.h"
 #include "base/utf_string_conversions.h"
 #include "base/win/pe_image.h"
 #include "base/win/scoped_comptr.h"
 #include "base/win/scoped_handle.h"
 #include "base/win/win_util.h"
@@ skipping 1135 matching lines @@
     while (offset < actual_size_to_read) {
       uint8 unused = *(touch + offset);
       offset += step_size;
     }
     FreeLibrary(dll_module);
   }
 
   return true;
 }
 
+namespace internal {

[chrisha, 2012/01/26 18:23:09]

namespace internal is normally for .h files. Why not simply an anonymous
namespace?

[Roger McFarlane (Google), 2012/01/27 05:24:08]

Because then they wouldn't be accessible for unittesting (the PartialPreRead*
helpers, I mean). The other's could certainly be put into an anonymous
namespace.

+
+struct ScopedPtrVirtualFree {
+  void operator() (void* ptr) {
+    ::VirtualFree(ptr, 0, MEM_RELEASE);
+  }
+};
+
+bool SetFilePointer(HANDLE file_handle, size_t position) {
+  return position <= std::numeric_limits<LONG>::max() &&
+      ::SetFilePointer(file_handle,
+                       static_cast<LONG>(position),
+                       NULL,
+                       FILE_BEGIN) != INVALID_SET_FILE_POINTER;
+}
+
+bool ReadNextBytes(HANDLE file_handle, void* buffer, size_t bytes_to_read) {
+  DWORD bytes_read = 0;
+  return bytes_to_read <= std::numeric_limits<DWORD>::max() &&
+      ::ReadFile(file_handle,
+                 buffer,
+                 static_cast<DWORD>(bytes_to_read),
+                 &bytes_read,
+                 NULL) &&
+      bytes_read == bytes_to_read;
+}
+
+BASE_EXPORT bool PartialPreReadPeFile(const wchar_t* file_path,
+                                      size_t percentage,
+                                      size_t max_chunk_size) {
+  // TODO(rogerm): change this to be driven by a table within the PE file
+  //     (defaulting to full read if it's not there?) that's initialized
+  //     by the optimization toolchain.

[chrisha, 2012/01/26 18:23:09]

Maybe break this down further? A PreReadChunk function that takes base address,
offset and length? This could then be called per section by this wrapper
function, and another that looks up percentages from somewhere in the PE file.

[Roger McFarlane (Google), 2012/01/27 05:24:08]

Refactored along the lines you suggest. I haven't gone all the way to looking up
percentages in the PE file yet (left as a TODO).

+  DCHECK(file_path != NULL);
+
+  if (percentage == 0)
+    return true;
+
+  // Validate/setup the percentage and max_chunk_size.
+  const size_t kOneHundredPercent = 100;
+  const size_t kMinChunkSize = 1024 * 1024;
+  percentage = std::min(percentage, kOneHundredPercent);
+  max_chunk_size = std::max(max_chunk_size, kMinChunkSize);
+
+  // On Windows Vista and higher, we sequentially read file contents with an

[Sigurður Ásgeirsson, 2012/01/26 18:59:20]

copy/paste?

[Roger McFarlane (Google), 2012/01/27 05:24:08]

At some point the comment still seemed relevant.

Removed.

+  // optional cap on total percentage to read.
+  base::win::ScopedHandle file(
+      CreateFile(file_path,
+                 GENERIC_READ,
+                 FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+                 NULL,
+                 OPEN_EXISTING,
+                 FILE_FLAG_SEQUENTIAL_SCAN,
+                 NULL));
+
+  if (!file.IsValid())
+    return false;
+
+  // Allocate a resizable buffer for the headers. We initially reserve as much
+  // space as we typically see as the header size for chrome.dll.
+  const size_t kInitialBufferSize = 0x400;
+  std::string headers(kInitialBufferSize, char());

[Sigurður Ásgeirsson, 2012/01/26 18:59:20]

nit: I like std::vector<uint8> or such for buffers, as this makes it explicit
that we don't have string, per-se here.

[Roger McFarlane (Google), 2012/01/27 05:24:08]

Done.

+
+  // Read, hopefully, all of the headers.
+  if (!ReadNextBytes(file, &headers[0], headers.size()))
+    return false;
+
+  // The DOS headers start at offset 0, which allow us to get the offset of the
+  // NT headers. Let's ensure we've read enough to capture the NT headers.
+  size_t nt_headers_start =
+      reinterpret_cast<IMAGE_DOS_HEADER*>(&headers[0])->e_lfanew;
+  size_t nt_headers_end = nt_headers_start + sizeof(IMAGE_NT_HEADERS);
+  if (nt_headers_end > headers.size()) {
+    size_t current_length = headers.size();
+    size_t bytes_to_read = nt_headers_end - current_length;
+    headers.resize(nt_headers_end);
+    if (!ReadNextBytes(file, &headers[current_length], bytes_to_read))
+      return false;
+  }
+
+  // Now that we've got the NT headers we can get the total header size,
+  // including all of the section headers. Let's ensure we've read enough
+  // to capture all of the section headers.
+  size_t size_of_headers = reinterpret_cast<IMAGE_NT_HEADERS*>(
+      &headers[nt_headers_start])->OptionalHeader.SizeOfHeaders;
+  if (size_of_headers > headers.size()) {
+    size_t current_length = headers.size();
+    size_t bytes_to_read = size_of_headers - current_length;
+    headers.resize(size_of_headers);
+    if (!ReadNextBytes(file, &headers[current_length], bytes_to_read))
+      return false;
+  }
+
+  // Now we have all of the headers.
+  const IMAGE_DOS_HEADER* dos_header =
+      reinterpret_cast<IMAGE_DOS_HEADER*>(&headers[0]);
+  const IMAGE_NT_HEADERS* nt_headers =
+      reinterpret_cast<IMAGE_NT_HEADERS*>(&headers[dos_header->e_lfanew]);
+  const IMAGE_SECTION_HEADER* section_headers = IMAGE_FIRST_SECTION(nt_headers);

[chrisha, 2012/01/26 18:23:09]

Could break out the header reading functionality to a utility function?

[Sigurður Ásgeirsson, 2012/01/26 18:59:20]

You can construct a PEImage instance on the buffer, and use it to access the
headers.

[Roger McFarlane (Google), 2012/01/27 05:24:08]

Done. Thanks!

[Roger McFarlane (Google), 2012/01/27 05:24:08]

I've done some refactoring, PTAL.

+
+  // Allocate a buffer to hold the pre-read bytes.
+  scoped_ptr_malloc<uint8, ScopedPtrVirtualFree> buffer(
+      reinterpret_cast<uint8*>(
+          ::VirtualAlloc(NULL, max_chunk_size, MEM_COMMIT, PAGE_READWRITE)));
+  if (buffer.get() == NULL)
+    return false;
+
+  // Iterate over each section, reading in a percentage of each.
+  for (size_t i = 0; i < nt_headers->FileHeader.NumberOfSections; ++i) {
+    if (!SetFilePointer(file, section_headers[i].PointerToRawData))
+      return false;
+    size_t max_bytes =
+        (section_headers[i].SizeOfRawData * percentage) / kOneHundredPercent;
+    while (max_bytes > 0) {
+      size_t chunk_size = std::min(max_bytes, max_chunk_size);
+      if (!ReadNextBytes(file, buffer.get(), chunk_size))
+        return false;
+      max_bytes -= chunk_size;
+    }
+  }
+
+  // We're done.
+  return true;
+}
+
+BASE_EXPORT bool PartialPreReadPeImage(const wchar_t* file_path,
+                                       size_t percentage,
+                                       size_t step_size) {
+  // TODO(rogerm): change this to be driven by a table within the PE image
+  //     (defaulting to full read if it's not there?) that's initialized
+  //     by the optimization toolchain.
+  DCHECK(file_path != NULL);
+
+  if (percentage == 0)
+    return true;
+
+  // Validate/setup the percentage and step_size.
+  const size_t kOneHundredPercent = 100;
+  const size_t kMaxStepSize = 4096;
+  percentage = std::min(percentage, kOneHundredPercent);
+  step_size = std::min(step_size, kMaxStepSize);

[Sigurður Ásgeirsson, 2012/01/26 18:59:20]

I don't know that it's worth parametrizing the stride, as we'll always use
PAGE_SIZE?

[Roger McFarlane (Google), 2012/01/27 05:24:08]

Done. Switched to always using the system page size.

+
+  // WinXP branch. Here, reading the DLL from disk doesn't do

[Sigurður Ásgeirsson, 2012/01/26 18:59:20]

copy/paste?

[Roger McFarlane (Google), 2012/01/27 05:24:08]

At some point the comment still seemed relevant.

Removed.

+  // what we want so instead we pull the pages into memory by loading
+  // the DLL and touching pages at a stride.
+  HMODULE dll_module = ::LoadLibraryExW(
+      file_path,
+      NULL,
+      LOAD_WITH_ALTERED_SEARCH_PATH | DONT_RESOLVE_DLL_REFERENCES);
+
+  if (!dll_module)
+    return false;
+
+  // Iterate over each section, stepping through a percentage of each to page
+  // it in off the disk.
+  base::win::PEImage pe_image(dll_module);

[Sigurður Ásgeirsson, 2012/01/26 18:59:20]

May want to CHECK(pe_image.VerifyMagic())?

[Roger McFarlane (Google), 2012/01/27 05:24:08]

Done.

+  for (UINT i = 0; i < pe_image.GetNumSections(); ++i) {
+    // Get the current section and figure out how much of it to touch.
+    const IMAGE_SECTION_HEADER* section = pe_image.GetSectionHeader(i);
+    DCHECK(section != NULL);
+    volatile uint8* touch =
+        reinterpret_cast<uint8*>(pe_image.RVAToAddr(section->VirtualAddress));
+    size_t section_length =

[Sigurður Ásgeirsson, 2012/01/26 18:59:20]

Maybe a one-line comment on why we don't only use VirtualSize?

[Roger McFarlane (Google), 2012/01/27 05:24:08]

Done.

+        std::min(section->SizeOfRawData, section->Misc.VirtualSize);
+    volatile uint8* max_touch =
+        touch + ((section_length * percentage) / kOneHundredPercent) - 1;
+
+    // Verify that the extent we're going to touch falls inside the section
+    // we expect it to (and by implication, inside the pe_image).
+    DCHECK_EQ(section,
+              pe_image.GetImageSectionFromAddr(const_cast<uint8*>(touch)));
+    DCHECK_EQ(section,
+              pe_image.GetImageSectionFromAddr(const_cast<uint8*>(max_touch)));
+
+    // Read the memory in the range [touch, max_touch] with a stride of
+    // step_size. This ensures that it's been paged in.
+    uint8 unused = *touch;
+    while (touch < max_touch) {
+      touch = std::max(touch + step_size, max_touch);

[Sigurður Ásgeirsson, 2012/01/26 18:59:20]

I don't understand, this would always return max_touch?

[Roger McFarlane (Google), 2012/01/27 05:24:08]

Oops, should have been std::min()... Nice catch!

That said, I've tweaked the loop a bit to get rid of the compare/branches inside
the loop.

+      unused = *touch;
+    }
+  }
+
+  FreeLibrary(dll_module);
+
+  return true;
+}
+
+}  // namespace file_util::internal
+
+bool PartialPreReadImage(const wchar_t* file_path,

[chrisha, 2012/01/26 18:23:09]

Confusing function names: PartialPreReadImage vs PartialPreReadPeImage.

[Roger McFarlane (Google), 2012/01/27 05:24:08]

Renamed. Are the new names better?

+                         size_t percentage,
+                         size_t step_size) {
+  base::ThreadRestrictions::AssertIOAllowed();
+
+  if (base::win::GetVersion() > base::win::VERSION_XP) {
+    // Vista+ branch. On these OSes, the forced reads through the DLL actually
+    // slows warm starts. The solution is to sequentially read file contents
+    // with an optional cap on total amount to read.
+    return internal::PartialPreReadPeFile(file_path, percentage, step_size);
+  }
+
+  // WinXP branch. Here, reading the DLL from disk doesn't do what we want
+  // so instead we pull the pages into memory by loading the DLL and touching
+  // pages at a stride.
+  return internal::PartialPreReadPeImage(file_path, percentage, step_size);
+}
+
 }  // namespace file_util