Switch to the good LRU implementation.

third_party/buildbot_8_4p1/buildbot/status/builder.py

 # This file is part of Buildbot.  Buildbot is free software: you can
 # redistribute it and/or modify it under the terms of the GNU General Public
 # License as published by the Free Software Foundation, version 2.
 #
 # This program is distributed in the hope that it will be useful, but WITHOUT
 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 # details.
 #
 # You should have received a copy of the GNU General Public License along with
@@ skipping 68 matching lines @@
         self.slavenames = []
         self.events = []
         # these three hold Events, and are used to retrieve the current
         # state of the boxes.
         self.lastBuildStatus = None
         #self.currentBig = None
         #self.currentSmall = None
         self.currentBuilds = []
         self.nextBuild = None
         self.watchers = []
-        self.buildCache = weakref.WeakValueDictionary()
-        self.buildCache_LRU = []
+        #self.buildCache = weakref.WeakValueDictionary()

[M-A Ruel, 2012/03/16 13:02:06]

I'd simply remove the old lines instead of commenting them.

[cmp, 2012/03/16 17:40:51]

Agree.

[szager, 2012/03/16 20:52:04]

Done.

+        #self.buildCache_LRU = []
+        self.buildCache = util.lru.AsyncLRUCache(self.cacheMiss,
+                                                 self.buildCacheSize)
         self.logCompressionLimit = False # default to no compression for tests
         self.logCompressionMethod = "bz2"
         self.logMaxSize = None # No default limit
         self.logMaxTailSize = None # No tail buffering
 
     # persistence
 
     def __getstate__(self):
         # when saving, don't record transient stuff like what builds are
         # currently running, because they won't be there when we start back
         # up. Nor do we save self.watchers, nor anything that gets set by our
         # parent like .basedir and .status
         d = styles.Versioned.__getstate__(self)
         d['watchers'] = []
         del d['buildCache']
-        del d['buildCache_LRU']
+        #del d['buildCache_LRU']
         for b in self.currentBuilds:
             b.saveYourself()
             # TODO: push a 'hey, build was interrupted' event
         del d['currentBuilds']
         d.pop('pendingBuilds', None)
         del d['currentBigState']
         del d['basedir']
         del d['status']
         del d['nextBuildNumber']
         return d
 
     def __setstate__(self, d):
         # when loading, re-initialize the transient stuff. Remember that
         # upgradeToVersion1 and such will be called after this finishes.
         styles.Versioned.__setstate__(self, d)
-        self.buildCache = weakref.WeakValueDictionary()
-        self.buildCache_LRU = []
+        #self.buildCache = weakref.WeakValueDictionary()
+        #self.buildCache_LRU = []
+        self.buildCache = util.lru.AsyncLRUCache(self.cacheMiss,
+                                                 self.buildCacheSize)
         self.currentBuilds = []
         self.watchers = []
         self.slavenames = []
         # self.basedir must be filled in by our parent
         # self.status must be filled in by our parent
 
     def reconfigFromBuildmaster(self, buildmaster):
         # Note that we do not hang onto the buildmaster, since this object
         # gets pickled and unpickled.
         if buildmaster.buildCacheSize is not None:
             self.buildCacheSize = buildmaster.buildCacheSize
+            self.buildCache.set_max_size(buildmaster.buildCacheSize)
 
     def upgradeToVersion1(self):
         if hasattr(self, 'slavename'):
             self.slavenames = [self.slavename]
             del self.slavename
         if hasattr(self, 'nextBuildNumber'):
             del self.nextBuildNumber # determineNextBuildNumber chooses this
         self.wasUpgraded = True
 
     def determineNextBuildNumber(self):
@@ skipping 34 matching lines @@
         try:
             dump(self, open(tmpfilename, "wb"), -1)
             if runtime.platformType  == 'win32':
                 # windows cannot rename a file on top of an existing one
                 if os.path.exists(filename):
                     os.unlink(filename)
             os.rename(tmpfilename, filename)
         except:
             log.msg("unable to save builder %s" % self.name)
             log.err()
-        
+
 
     # build cache management
 
     def makeBuildFilename(self, number):
         return os.path.join(self.basedir, "%d" % number)
 
-    def touchBuildCache(self, build):
-        self.buildCache[build.number] = build
-        if build in self.buildCache_LRU:
-            self.buildCache_LRU.remove(build)
-        self.buildCache_LRU = self.buildCache_LRU[-(self.buildCacheSize-1):] + [ build ]
-        return build
+#    def touchBuildCache(self, build):
+#        self.buildCache[build.number] = build
+#        if build in self.buildCache_LRU:
+#            self.buildCache_LRU.remove(build)
+#        self.buildCache_LRU = self.buildCache_LRU[-(self.buildCacheSize-1):] + [ build ]
+#        return build
 
     def getBuildByNumber(self, number):
-        # first look in currentBuilds
-        for b in self.currentBuilds:
-            if b.number == number:
-                return self.touchBuildCache(b)
+        return self.buildCache.get(number)
 
-        # then in the buildCache
-        if number in self.buildCache:
-            metrics.MetricCountEvent.log("buildCache.hits", 1)
-            return self.touchBuildCache(self.buildCache[number])
-        metrics.MetricCountEvent.log("buildCache.misses", 1)
-
-        # then fall back to loading it from disk
+    def loadBuildFromFile(self, number):
         filename = self.makeBuildFilename(number)
         try:
             log.msg("Loading builder %s's build %d from on-disk pickle"
                 % (self.name, number))
             build = load(open(filename, "rb"))
             build.builder = self
 
             # (bug #1068) if we need to upgrade, we probably need to rewrite
             # this pickle, too.  We determine this by looking at the list of
             # Versioned objects that have been unpickled, and (after doUpgrade)
             # checking to see if any of them set wasUpgraded.  The Versioneds'
             # upgradeToVersionNN methods all set this.
             versioneds = styles.versionedsToUpgrade
             styles.doUpgrade()
             if True in [ hasattr(o, 'wasUpgraded') for o in versioneds.values() ]:
                 log.msg("re-writing upgraded build pickle")
                 build.saveYourself()
 
             # handle LogFiles from after 0.5.0 and before 0.6.5
             build.upgradeLogfiles()
             # check that logfiles exist
             build.checkLogfiles()
-            return self.touchBuildCache(build)
+            #return self.touchBuildCache(build)
+            return build
         except IOError:
             raise IndexError("no such build %d" % number)
         except EOFError:
             raise IndexError("corrupted build pickle %d" % number)
 
+    def cacheMiss(self, number):
+        # first look in currentBuilds
+        for b in self.currentBuilds:

[M-A Ruel, 2012/03/16 13:02:06]

if any(b.number == number for b in self.currentBuilds):

[szager, 2012/03/16 20:52:04]

I don't like that because it requires two linear searches through
self.currentBuilds -- once for 'any', and then again to return the value.

+            if b.number == number:
+                #return self.touchBuildCache(b)
+                return defer.succeed(b)
+
+        # then fall back to loading it from disk
+        return threads.deferToThread(self.loadBuildFromFile, number)
+
     def prune(self, events_only=False):
         # begin by pruning our own events
         self.events = self.events[-self.eventHorizon:]
 
         if events_only:
             return
 
         gc.collect()
 
         # get the horizons straight
@@ skipping 26 matching lines @@
             is_logfile = False
             if mo:
                 num = int(mo.group(1))
             else:
                 mo = build_log_re.match(filename)
                 if mo:
                     num = int(mo.group(1))
                     is_logfile = True
 
             if num is None: continue
-            if num in self.buildCache: continue
+            if num in self.buildCache.cache: continue
 
             if (is_logfile and num < earliest_log) or num < earliest_build:
                 pathname = os.path.join(self.basedir, filename)
                 log.msg("pruning '%s'" % pathname)
                 try: os.unlink(pathname)
                 except OSError: pass
 
     # IBuilderStatus methods
     def getName(self):
         return self.name
@@ skipping 202 matching lines @@
 
     # buildStarted is called by our child BuildStatus instances
     def buildStarted(self, s):
         """Now the BuildStatus object is ready to go (it knows all of its
         Steps, its ETA, etc), so it is safe to notify our watchers."""
 
         assert s.builder is self # paranoia
         assert s.number == self.nextBuildNumber - 1
         assert s not in self.currentBuilds
         self.currentBuilds.append(s)
-        self.touchBuildCache(s)
+        #self.touchBuildCache(s)
+        self.buildCache.put(s.number, s)
 
         # now that the BuildStatus is prepared to answer queries, we can
         # announce the new build to all our watchers
 
         for w in self.watchers: # TODO: maybe do this later? callLater(0)?
             try:
                 receiver = w.buildStarted(self.getName(), s)
                 if receiver:
                     if type(receiver) == type(()):
                         s.subscribe(receiver[0], receiver[1])
@@ skipping 89 matching lines @@
         result['basedir'] = os.path.basename(self.basedir)
         result['category'] = self.category
         result['slaves'] = self.slavenames
         #result['url'] = self.parent.getURLForThing(self)
         # TODO(maruel): Add cache settings? Do we care?
 
         # Transient
         # Collect build numbers.
         # Important: Only grab the *cached* builds numbers to reduce I/O.
         current_builds = [b.getNumber() for b in self.currentBuilds]
-        cached_builds = list(set(self.buildCache.keys() + current_builds))
+        cached_builds = list(set(self.buildCache.cache.keys() + current_builds))
         cached_builds.sort()
         result['cachedBuilds'] = cached_builds
         result['currentBuilds'] = current_builds
         result['state'] = self.getState()[0]
         # lies, but we don't have synchronous access to this info; use
         # asDict_async instead
         result['pendingBuilds'] = 0
         return result
 
     def asDict_async(self):
         """Just like L{asDict}, but with a nonzero pendingBuilds."""
         result = self.asDict()
         d = self.getPendingBuildRequestStatuses()
         def combine(statuses):
             result['pendingBuilds'] = len(statuses)
             return result
         d.addCallback(combine)
         return d
 
     def getMetrics(self):
         return self.botmaster.parent.metrics
 
 # vim: set ts=4 sts=4 sw=4 et: