Extensions Docs Server: Large performance increase

chrome/common/extensions/docs/server2/memcache_file_system_test.py

Index: chrome/common/extensions/docs/server2/memcache_file_system_test.py
diff --git a/chrome/common/extensions/docs/server2/memcache_file_system_test.py b/chrome/common/extensions/docs/server2/memcache_file_system_test.py
index 6f89b4888e3c80aaa69549c0d06c125550da9298..712fe65baf197ba10ea9bc9ba17881b7f98d9860 100755
--- a/chrome/common/extensions/docs/server2/memcache_file_system_test.py
+++ b/chrome/common/extensions/docs/server2/memcache_file_system_test.py
@@ -6,9 +6,9 @@
 import os
 import unittest
 
-import appengine_memcache as memcache
-from appengine_memcache import AppEngineMemcache
-from file_system import FileSystem
+import object_store
+from in_memory_object_store import InMemoryObjectStore
+from file_system import FileSystem, StatInfo
 from future import Future
 from local_file_system import LocalFileSystem
 from memcache_file_system import MemcacheFileSystem
@@ -29,9 +29,11 @@ class _FakeFileSystem(FileSystem):
 
   def Stat(self, path):
     self._stat_count += 1
-    return self.StatInfo(
-        self.stat_value,
-        dict((path + str(i), self.stat_value) for i in range(5)))
+    children = dict((path.strip('/') + str(i), self.stat_value)
+                    for i in range(5))
+    if not path.endswith('/'):
+      children[path.rsplit('/', 1)[-1]] = self.stat_value
+    return StatInfo(self.stat_value, children)
 
   def Read(self, paths, binary=False):
     self._read_count += 1
@@ -39,34 +41,34 @@ class _FakeFileSystem(FileSystem):
 
 class MemcacheFileSystemTest(unittest.TestCase):
   def setUp(self):
-    self._memcache = AppEngineMemcache('')
+    self._object_store = InMemoryObjectStore('')
     self._local_fs = LocalFileSystem(os.path.join('test_data', 'file_system'))
 
   def _SetReadCacheItem(self, key, value, stat):
-    self._memcache.Set(key, (value, stat), memcache.MEMCACHE_FILE_SYSTEM_READ)
+    self._object_store.Set(key, (value, stat), object_store.FILE_SYSTEM_READ)
 
   def _SetStatCacheItem(self, key, value):
-    self._memcache.Set(key, value, memcache.MEMCACHE_FILE_SYSTEM_STAT)
+    self._object_store.Set(key, value, object_store.FILE_SYSTEM_STAT)
 
   def _DeleteReadCacheItem(self, key):
-    self._memcache.Delete(key, memcache.MEMCACHE_FILE_SYSTEM_READ)
+    self._object_store.Delete(key, object_store.FILE_SYSTEM_READ)
 
   def _DeleteStatCacheItem(self, key):
-    self._memcache.Delete(key, memcache.MEMCACHE_FILE_SYSTEM_STAT)
+    self._object_store.Delete(key, object_store.FILE_SYSTEM_STAT)
 
   def testReadFiles(self):
-    file_system = MemcacheFileSystem(self._local_fs, self._memcache)
+    file_system = MemcacheFileSystem(self._local_fs, self._object_store)
     expected = {
-      'test1.txt': 'test1\n',
-      'test2.txt': 'test2\n',
-      'test3.txt': 'test3\n',
+      './test1.txt': 'test1\n',
+      './test2.txt': 'test2\n',
+      './test3.txt': 'test3\n',
     }
     self.assertEqual(
         expected,
-        file_system.Read(['test1.txt', 'test2.txt', 'test3.txt']).Get())
+        file_system.Read(['./test1.txt', './test2.txt', './test3.txt']).Get())
 
   def testListDir(self):
-    file_system = MemcacheFileSystem(self._local_fs, self._memcache)
+    file_system = MemcacheFileSystem(self._local_fs, self._object_store)
     expected = ['dir/']
     for i in range(7):
       expected.append('file%d.html' % i)
@@ -80,46 +82,51 @@ class MemcacheFileSystemTest(unittest.TestCase):
 
   def testCaching(self):
     fake_fs = _FakeFileSystem()
-    file_system = MemcacheFileSystem(fake_fs, self._memcache)
-    self.assertEqual('bob', file_system.ReadSingle('bob'))
+    file_system = MemcacheFileSystem(fake_fs, self._object_store)
+    self.assertEqual('bob/bob0', file_system.ReadSingle('bob/bob0'))
     self.assertTrue(fake_fs.CheckAndReset(read_count=1, stat_count=1))
 
-    # Resource has been memcached, so test resource is not re-fetched.
-    self.assertEqual('bob', file_system.ReadSingle('bob'))
+    # Resource has been cached, so test resource is not re-fetched.
+    self.assertEqual('bob/bob0', file_system.ReadSingle('bob/bob0'))
     self.assertTrue(fake_fs.CheckAndReset())
 
     # Test if the Stat version is the same the resource is not re-fetched.
-    self._DeleteStatCacheItem('bob')
-    self.assertEqual('bob', file_system.ReadSingle('bob'))
+    self._DeleteStatCacheItem('bob/bob0')
+    self.assertEqual('bob/bob0', file_system.ReadSingle('bob/bob0'))
     self.assertTrue(fake_fs.CheckAndReset(stat_count=1))
 
     # Test if there is a newer version, the resource is re-fetched.
-    self._DeleteStatCacheItem('bob')
+    self._DeleteStatCacheItem('bob/bob0')
     fake_fs.stat_value += 1
-    self.assertEqual('bob', file_system.ReadSingle('bob'))
+    self.assertEqual('bob/bob0', file_system.ReadSingle('bob/bob0'))
     self.assertTrue(fake_fs.CheckAndReset(read_count=1, stat_count=1))
 
     # Test directory and subdirectory stats are cached.
-    self.assertEqual('bob/', file_system.ReadSingle('bob/'))
+    self._DeleteStatCacheItem('bob/bob0')
+    self._DeleteReadCacheItem('bob/bob0')
+    self._DeleteStatCacheItem('bob/bob1')
+    self.assertEqual('bob/bob1', file_system.ReadSingle('bob/bob1'))
     self.assertEqual('bob/bob0', file_system.ReadSingle('bob/bob0'))
     self.assertTrue(fake_fs.CheckAndReset(read_count=2, stat_count=1))
-    self.assertEqual('bob/', file_system.ReadSingle('bob/'))
+    self.assertEqual('bob/bob1', file_system.ReadSingle('bob/bob1'))
     self.assertTrue(fake_fs.CheckAndReset())
 
     # Test a more recent parent directory doesn't force a refetch of children.
+    self._DeleteReadCacheItem('bob/bob0')
+    self._DeleteReadCacheItem('bob/bob1')
     self.assertEqual('bob/bob1', file_system.ReadSingle('bob/bob1'))
     self.assertEqual('bob/bob2', file_system.ReadSingle('bob/bob2'))
     self.assertEqual('bob/bob3', file_system.ReadSingle('bob/bob3'))
     self.assertTrue(fake_fs.CheckAndReset(read_count=3))
     self._SetStatCacheItem('bob/', 10)
-    self.assertEqual('bob/bob0', file_system.ReadSingle('bob/bob0'))
     self.assertEqual('bob/bob1', file_system.ReadSingle('bob/bob1'))
     self.assertEqual('bob/bob2', file_system.ReadSingle('bob/bob2'))
+    self.assertEqual('bob/bob3', file_system.ReadSingle('bob/bob3'))
     self.assertTrue(fake_fs.CheckAndReset())
 
     self._DeleteStatCacheItem('bob/bob0')
     self.assertEqual('bob/bob0', file_system.ReadSingle('bob/bob0'))
-    self.assertTrue(fake_fs.CheckAndReset(stat_count=1))
+    self.assertTrue(fake_fs.CheckAndReset(read_count=1, stat_count=1))
     self.assertEqual('bob/bob0', file_system.ReadSingle('bob/bob0'))
     self.assertTrue(fake_fs.CheckAndReset())