Extension docs server: many changes to bring down the latency of the server,

chrome/common/extensions/docs/server2/memcache_file_system.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-from file_system import FileSystem
+import logging
+import time
+
+from file_system import FileSystem, StatInfo
 from future import Future
 import appengine_memcache as memcache
 
 class MemcacheFileSystem(FileSystem):
   """FileSystem implementation which memcaches the results of Read.
   """
   def __init__(self, file_system, memcache):
     self._file_system = file_system
     self._memcache = memcache
 
   def Stat(self, path):
+    start_time = time.time()
+    try:
+      return self._DoStat(path)
+    finally:
+      logging.info("(cached) Stat of %s took %sms",
+                   path,
+                   (time.time() - start_time) * 1000)
+
+  def _DoStat(self, path):
     """Stats the directory given, or if a file is given, stats the files parent
     directory to get info about the file.
     """
+    # TODO(kalman): store the whole stat info, not just the version.
     version = self._memcache.Get(path, memcache.MEMCACHE_FILE_SYSTEM_STAT)
-    if version is None:
-      stat_info = self._file_system.Stat(path)
-      self._memcache.Set(path,
-                         stat_info.version,
+    if version is not None:
+      return StatInfo(version)
+
+    # Always stat the parent directory, since it will have the stat of the child
+    # anyway, and this gives us an entire directory's stat info at once.
+    if path.endswith('/'):
+      dir_path = path
+    else:
+      dir_path = path.rsplit('/', 1)[0] + '/'
+
+    # TODO: batch set?
+    dir_stat = self._file_system.Stat(dir_path)
+    if path == dir_path:
+      version = dir_stat.version
+
+    self._memcache.Set(path,
+                       dir_stat.version,
+                       memcache.MEMCACHE_FILE_SYSTEM_STAT)
+
+    for child_path, child_version in dir_stat.child_versions.iteritems():
+      child_path = dir_path + child_path
+      if path == child_path:
+        version = child_version
+      self._memcache.Set(child_path,
+                         child_version,
                          memcache.MEMCACHE_FILE_SYSTEM_STAT)
-      if stat_info.child_versions is not None:
-        for child_path, child_version in stat_info.child_versions.iteritems():
-          self._memcache.Set(path.rsplit('/', 1)[0] + '/' + child_path,
-                             child_version,
-                             memcache.MEMCACHE_FILE_SYSTEM_STAT)
-    else:
-      stat_info = self.StatInfo(version)
-    return stat_info
+    return StatInfo(version)
 
   def Read(self, paths, binary=False):
+    start_time = time.time()
+    try:
+      return self._DoRead(paths, binary=binary)
+    finally:
+      logging.info("(cached) Read of %s took %sms",
+                   paths,
+                   (time.time() - start_time) * 1000)
+
+  def _DoRead(self, paths, binary=False):
     """Reads a list of files. If a file is in memcache and it is not out of
     date, it is returned. Otherwise, the file is retrieved from the file system.
     """
     result = {}
     uncached = []
     for path in paths:
       cached_result = self._memcache.Get(path,
                                          memcache.MEMCACHE_FILE_SYSTEM_READ)
       if cached_result is None:
         uncached.append(path)
         continue
       data, version = cached_result
       if self.Stat(path).version != version:
         self._memcache.Delete(path, memcache.MEMCACHE_FILE_SYSTEM_READ)
         uncached.append(path)
         continue
       result[path] = data
-    if uncached:
-      # TODO(cduvall): if there are uncached items we should return an
-      # asynchronous future. http://crbug.com/142013
-      new_items = self._file_system.Read(uncached, binary=binary).Get()
-      for item in new_items:
-        version = self.Stat(item).version
-        value = new_items[item]
-        # Cache this file forever.
-        self._memcache.Set(item,
-                           (value, version),
-                           memcache.MEMCACHE_FILE_SYSTEM_READ,
-                           time=0)
-        result[item] = value
+
+    if not uncached:
+      return Future(value=result)
+
+    # TODO(cduvall): if there are uncached items we should return an
+    # asynchronous future. http://crbug.com/142013
+    new_items = self._file_system.Read(uncached, binary=binary).Get()
+    for item in new_items:
+      version = self.Stat(item).version
+      value = new_items[item]
+      self._memcache.Set(item,
+                         (value, version),
+                         memcache.MEMCACHE_FILE_SYSTEM_READ,
+                         # Forever.
+                         time=0)
+      result[item] = value
     return Future(value=result)