OLD | NEW |
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import posixpath | 5 import posixpath |
6 import sys | 6 import sys |
7 | 7 |
8 from file_system import FileSystem, StatInfo, FileNotFoundError | 8 from file_system import FileSystem, StatInfo, FileNotFoundError |
9 from future import Future | 9 from future import Future |
10 from path_util import IsDirectory | 10 from path_util import IsDirectory |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
56 file_version = dir_stat.child_versions.get(file_path) | 56 file_version = dir_stat.child_versions.get(file_path) |
57 if file_version is None: | 57 if file_version is None: |
58 raise FileNotFoundError('No stat found for %s in %s (found %s)' % | 58 raise FileNotFoundError('No stat found for %s in %s (found %s)' % |
59 (path, dir_path, dir_stat.child_versions)) | 59 (path, dir_path, dir_stat.child_versions)) |
60 return StatInfo(file_version) | 60 return StatInfo(file_version) |
61 | 61 |
62 dir_stat = self._stat_object_store.Get(dir_path).Get() | 62 dir_stat = self._stat_object_store.Get(dir_path).Get() |
63 if dir_stat is not None: | 63 if dir_stat is not None: |
64 return Future(value=make_stat_info(dir_stat)) | 64 return Future(value=make_stat_info(dir_stat)) |
65 | 65 |
66 dir_stat_future = self._MemoizedStatAsyncFromFileSystem(dir_path) | 66 def next(dir_stat): |
67 def resolve(): | |
68 dir_stat = dir_stat_future.Get() | |
69 assert dir_stat is not None # should have raised a FileNotFoundError | 67 assert dir_stat is not None # should have raised a FileNotFoundError |
70 # We only ever need to cache the dir stat. | 68 # We only ever need to cache the dir stat. |
71 self._stat_object_store.Set(dir_path, dir_stat) | 69 self._stat_object_store.Set(dir_path, dir_stat) |
72 return make_stat_info(dir_stat) | 70 return make_stat_info(dir_stat) |
73 return Future(callback=resolve) | 71 return self._MemoizedStatAsyncFromFileSystem(dir_path).Then(next) |
74 | 72 |
75 @memoize | 73 @memoize |
76 def _MemoizedStatAsyncFromFileSystem(self, dir_path): | 74 def _MemoizedStatAsyncFromFileSystem(self, dir_path): |
77 '''This is a simple wrapper to memoize Futures to directory stats, since | 75 '''This is a simple wrapper to memoize Futures to directory stats, since |
78 StatAsync makes heavy use of it. Only cache directories so that the | 76 StatAsync makes heavy use of it. Only cache directories so that the |
79 memoized cache doesn't blow up. | 77 memoized cache doesn't blow up. |
80 ''' | 78 ''' |
81 assert IsDirectory(dir_path) | 79 assert IsDirectory(dir_path) |
82 return self._file_system.StatAsync(dir_path) | 80 return self._file_system.StatAsync(dir_path) |
83 | 81 |
84 def Read(self, paths, skip_not_found=False): | 82 def Read(self, paths, skip_not_found=False): |
85 '''Reads a list of files. If a file is in memcache and it is not out of | 83 '''Reads a list of files. If a file is in memcache and it is not out of |
86 date, it is returned. Otherwise, the file is retrieved from the file system. | 84 date, it is returned. Otherwise, the file is retrieved from the file system. |
87 ''' | 85 ''' |
88 cached_read_values = self._read_object_store.GetMulti(paths).Get() | 86 cached_read_values = self._read_object_store.GetMulti(paths).Get() |
89 cached_stat_values = self._stat_object_store.GetMulti(paths).Get() | 87 cached_stat_values = self._stat_object_store.GetMulti(paths).Get() |
90 | 88 |
91 # Populate a map of paths to Futures to their stat. They may have already | 89 # Populate a map of paths to Futures to their stat. They may have already |
92 # been cached in which case their Future will already have been constructed | 90 # been cached in which case their Future will already have been constructed |
93 # with a value. | 91 # with a value. |
94 stat_futures = {} | 92 stat_futures = {} |
95 | 93 |
96 def swallow_file_not_found_error(future): | 94 def handle(error): |
97 def resolve(): | 95 if isinstance(error, FileNotFoundError): |
98 try: return future.Get() | 96 return None |
99 except FileNotFoundError: return Nnone | 97 raise error |
100 return Future(callback=resolve) | |
101 | 98 |
102 for path in paths: | 99 for path in paths: |
103 stat_value = cached_stat_values.get(path) | 100 stat_value = cached_stat_values.get(path) |
104 if stat_value is None: | 101 if stat_value is None: |
105 stat_future = self.StatAsync(path) | 102 stat_future = self.StatAsync(path) |
106 if skip_not_found: | 103 if skip_not_found: |
107 stat_future = swallow_file_not_found_error(stat_future) | 104 stat_future = stat_future.Then(lambda x: x, handle) |
108 else: | 105 else: |
109 stat_future = Future(value=stat_value) | 106 stat_future = Future(value=stat_value) |
110 stat_futures[path] = stat_future | 107 stat_futures[path] = stat_future |
111 | 108 |
112 # Filter only the cached data which is fresh by comparing to the latest | 109 # Filter only the cached data which is fresh by comparing to the latest |
113 # stat. The cached read data includes the cached version. Remove it for | 110 # stat. The cached read data includes the cached version. Remove it for |
114 # the result returned to callers. | 111 # the result returned to callers. |
115 fresh_data = dict( | 112 fresh_data = dict( |
116 (path, data) for path, (data, version) in cached_read_values.iteritems() | 113 (path, data) for path, (data, version) in cached_read_values.iteritems() |
117 if stat_futures[path].Get().version == version) | 114 if stat_futures[path].Get().version == version) |
118 | 115 |
119 if len(fresh_data) == len(paths): | 116 if len(fresh_data) == len(paths): |
120 # Everything was cached and up-to-date. | 117 # Everything was cached and up-to-date. |
121 return Future(value=fresh_data) | 118 return Future(value=fresh_data) |
122 | 119 |
123 # Read in the values that were uncached or old. | 120 def next(new_results): |
124 read_futures = self._file_system.Read( | |
125 set(paths) - set(fresh_data.iterkeys()), | |
126 skip_not_found=skip_not_found) | |
127 def resolve(): | |
128 new_results = read_futures.Get() | |
129 # Update the cache. This is a path -> (data, version) mapping. | 121 # Update the cache. This is a path -> (data, version) mapping. |
130 self._read_object_store.SetMulti( | 122 self._read_object_store.SetMulti( |
131 dict((path, (new_result, stat_futures[path].Get().version)) | 123 dict((path, (new_result, stat_futures[path].Get().version)) |
132 for path, new_result in new_results.iteritems())) | 124 for path, new_result in new_results.iteritems())) |
133 new_results.update(fresh_data) | 125 new_results.update(fresh_data) |
134 return new_results | 126 return new_results |
135 return Future(callback=resolve) | 127 # Read in the values that were uncached or old. |
| 128 return self._file_system.Read(set(paths) - set(fresh_data.iterkeys()), |
| 129 skip_not_found=skip_not_found).Then(next) |
136 | 130 |
137 def GetIdentity(self): | 131 def GetIdentity(self): |
138 return self._file_system.GetIdentity() | 132 return self._file_system.GetIdentity() |
139 | 133 |
140 def __repr__(self): | 134 def __repr__(self): |
141 return '%s of <%s>' % (type(self).__name__, repr(self._file_system)) | 135 return '%s of <%s>' % (type(self).__name__, repr(self._file_system)) |
OLD | NEW |