named_caches: make more resilient to corrupted state

client/named_cache.py

 # Copyright 2016 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """This file implements Named Caches."""
 
 import contextlib
 import logging
 import optparse
 import os
@@ skipping 45 matching lines @@
     """Opens NamedCaches for mutation operations, such as request or trim.
 
     Only on caller can open the cache manager at a time. If the same thread
     calls this function after opening it earlier, the call will deadlock.
 
     time_fn is a function that returns timestamp (float) and used to take
     timestamps when new caches are requested.
 
     Returns a context manager that must be closed as soon as possible.
     """
-    state_path = os.path.join(self.root_dir, u'state.json')
     with self._lock:
+      state_path = os.path.join(self.root_dir, u'state.json')
+      assert self._lru is None, 'acquired lock, but self._lru is not None'
       if os.path.isfile(state_path):
-        self._lru = lru.LRUDict.load(state_path)
-      else:
-        self._lru = lru.LRUDict()
+        try:
+          self._lru = lru.LRUDict.load(state_path)
+        except ValueError:
+          logging.exception('failed to load named cache state file')
+          logging.warning('deleting named caches')
+          file_path.rmtree(self.root_dir)
+      self._lru = self._lru or lru.LRUDict()
       if time_fn:
         self._lru.time_fn = time_fn
       try:
         yield
       finally:
         file_path.ensure_tree(self.root_dir)
         self._lru.save(state_path)
         self._lru = None
 
   def __len__(self):
@@ skipping 194 matching lines @@
   if options.named_cache_root:
     return CacheManager(os.path.abspath(options.named_cache_root))
   return None
 
 
 def _validate_named_cache_path(path):
   if os.path.isabs(path):
     raise Error('named cache path must not be absolute')
   if '..' in path.split(os.path.sep):
     raise Error('named cache path must not contain ".."')