OLD | NEW |
1 URL: http://buildbot.net/trac | 1 URL: http://buildbot.net/trac |
2 Version: 0.8.4p1 | 2 Version: 0.8.4p1 |
3 License: GNU General Public License (GPL) Version 2 | 3 License: GNU General Public License (GPL) Version 2 |
4 | 4 |
5 This is a forked copy of buildbot v0.8.4p1. | 5 This is a forked copy of buildbot v0.8.4p1. |
6 | 6 |
7 | 7 |
8 Add extra parameters to HttpStatusPush as a very basic authentication mechanism. | 8 Add extra parameters to HttpStatusPush as a very basic authentication mechanism. |
9 | 9 |
10 diff --git a/third_party/buildbot_8_4p1/buildbot/status/status_push.py b/third_p
arty/buildbot_8_4p1/buildbot/status/status_push.py | 10 diff --git a/third_party/buildbot_8_4p1/buildbot/status/status_push.py b/third_p
arty/buildbot_8_4p1/buildbot/status/status_push.py |
(...skipping 1181 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1192 + tmp = [] | 1192 + tmp = [] |
1193 + if isinstance(self.command, list): | 1193 + if isinstance(self.command, list): |
1194 + self._flattenList(tmp, self.command) | 1194 + self._flattenList(tmp, self.command) |
1195 + else: | 1195 + else: |
1196 + tmp = self.command | 1196 + tmp = self.command |
1197 + | 1197 + |
1198 + kwargs['command'] = tmp | 1198 + kwargs['command'] = tmp |
1199 kwargs['logfiles'] = self.logfiles | 1199 kwargs['logfiles'] = self.logfiles |
1200 | 1200 |
1201 # check for the usePTY flag | 1201 # check for the usePTY flag |
| 1202 |
| 1203 For performance, switch to the good LRU cache implementation. |
| 1204 |
| 1205 Index: buildbot/status/builder.py |
| 1206 =================================================================== |
| 1207 --- buildbot/status/builder.py (revision 127129) |
| 1208 +++ buildbot/status/builder.py (working copy) |
| 1209 @@ -86,8 +86,10 @@ |
| 1210 self.currentBuilds = [] |
| 1211 self.nextBuild = None |
| 1212 self.watchers = [] |
| 1213 - self.buildCache = weakref.WeakValueDictionary() |
| 1214 - self.buildCache_LRU = [] |
| 1215 + #self.buildCache = weakref.WeakValueDictionary() |
| 1216 + #self.buildCache_LRU = [] |
| 1217 + self.buildCache = util.lru.AsyncLRUCache(self.cacheMiss, |
| 1218 + self.buildCacheSize) |
| 1219 self.logCompressionLimit = False # default to no compression for tests |
| 1220 self.logCompressionMethod = "bz2" |
| 1221 self.logMaxSize = None # No default limit |
| 1222 @@ -103,7 +105,7 @@ |
| 1223 d = styles.Versioned.__getstate__(self) |
| 1224 d['watchers'] = [] |
| 1225 del d['buildCache'] |
| 1226 - del d['buildCache_LRU'] |
| 1227 + #del d['buildCache_LRU'] |
| 1228 for b in self.currentBuilds: |
| 1229 b.saveYourself() |
| 1230 # TODO: push a 'hey, build was interrupted' event |
| 1231 @@ -119,8 +121,10 @@ |
| 1232 # when loading, re-initialize the transient stuff. Remember that |
| 1233 # upgradeToVersion1 and such will be called after this finishes. |
| 1234 styles.Versioned.__setstate__(self, d) |
| 1235 - self.buildCache = weakref.WeakValueDictionary() |
| 1236 - self.buildCache_LRU = [] |
| 1237 + #self.buildCache = weakref.WeakValueDictionary() |
| 1238 + #self.buildCache_LRU = [] |
| 1239 + self.buildCache = util.lru.AsyncLRUCache(self.cacheMiss, |
| 1240 + self.buildCacheSize) |
| 1241 self.currentBuilds = [] |
| 1242 self.watchers = [] |
| 1243 self.slavenames = [] |
| 1244 @@ -132,6 +136,7 @@ |
| 1245 # gets pickled and unpickled. |
| 1246 if buildmaster.buildCacheSize is not None: |
| 1247 self.buildCacheSize = buildmaster.buildCacheSize |
| 1248 + self.buildCache.set_max_size(buildmaster.buildCacheSize) |
| 1249 |
| 1250 def upgradeToVersion1(self): |
| 1251 if hasattr(self, 'slavename'): |
| 1252 @@ -186,33 +191,24 @@ |
| 1253 except: |
| 1254 log.msg("unable to save builder %s" % self.name) |
| 1255 log.err() |
| 1256 - |
| 1257 |
| 1258 + |
| 1259 # build cache management |
| 1260 |
| 1261 def makeBuildFilename(self, number): |
| 1262 return os.path.join(self.basedir, "%d" % number) |
| 1263 |
| 1264 - def touchBuildCache(self, build): |
| 1265 - self.buildCache[build.number] = build |
| 1266 - if build in self.buildCache_LRU: |
| 1267 - self.buildCache_LRU.remove(build) |
| 1268 - self.buildCache_LRU = self.buildCache_LRU[-(self.buildCacheSize-1):] +
[ build ] |
| 1269 - return build |
| 1270 +# def touchBuildCache(self, build): |
| 1271 +# self.buildCache[build.number] = build |
| 1272 +# if build in self.buildCache_LRU: |
| 1273 +# self.buildCache_LRU.remove(build) |
| 1274 +# self.buildCache_LRU = self.buildCache_LRU[-(self.buildCacheSize-1):] +
[ build ] |
| 1275 +# return build |
| 1276 |
| 1277 def getBuildByNumber(self, number): |
| 1278 - # first look in currentBuilds |
| 1279 - for b in self.currentBuilds: |
| 1280 - if b.number == number: |
| 1281 - return self.touchBuildCache(b) |
| 1282 + return self.buildCache.get(number) |
| 1283 |
| 1284 - # then in the buildCache |
| 1285 - if number in self.buildCache: |
| 1286 - metrics.MetricCountEvent.log("buildCache.hits", 1) |
| 1287 - return self.touchBuildCache(self.buildCache[number]) |
| 1288 - metrics.MetricCountEvent.log("buildCache.misses", 1) |
| 1289 - |
| 1290 - # then fall back to loading it from disk |
| 1291 + def loadBuildFromFile(self, number): |
| 1292 filename = self.makeBuildFilename(number) |
| 1293 try: |
| 1294 log.msg("Loading builder %s's build %d from on-disk pickle" |
| 1295 @@ -235,12 +231,23 @@ |
| 1296 build.upgradeLogfiles() |
| 1297 # check that logfiles exist |
| 1298 build.checkLogfiles() |
| 1299 - return self.touchBuildCache(build) |
| 1300 + #return self.touchBuildCache(build) |
| 1301 + return build |
| 1302 except IOError: |
| 1303 raise IndexError("no such build %d" % number) |
| 1304 except EOFError: |
| 1305 raise IndexError("corrupted build pickle %d" % number) |
| 1306 |
| 1307 + def cacheMiss(self, number): |
| 1308 + # first look in currentBuilds |
| 1309 + for b in self.currentBuilds: |
| 1310 + if b.number == number: |
| 1311 + #return self.touchBuildCache(b) |
| 1312 + return defer.succeed(b) |
| 1313 + |
| 1314 + # then fall back to loading it from disk |
| 1315 + return threads.deferToThread(self.loadBuildFromFile, number) |
| 1316 + |
| 1317 def prune(self, events_only=False): |
| 1318 # begin by pruning our own events |
| 1319 self.events = self.events[-self.eventHorizon:] |
| 1320 @@ -287,7 +294,7 @@ |
| 1321 is_logfile = True |
| 1322 |
| 1323 if num is None: continue |
| 1324 - if num in self.buildCache: continue |
| 1325 + if num in self.buildCache.cache: continue |
| 1326 |
| 1327 if (is_logfile and num < earliest_log) or num < earliest_build: |
| 1328 pathname = os.path.join(self.basedir, filename) |
| 1329 @@ -510,7 +517,8 @@ |
| 1330 assert s.number == self.nextBuildNumber - 1 |
| 1331 assert s not in self.currentBuilds |
| 1332 self.currentBuilds.append(s) |
| 1333 - self.touchBuildCache(s) |
| 1334 + #self.touchBuildCache(s) |
| 1335 + self.buildCache.put(s.number, s) |
| 1336 |
| 1337 # now that the BuildStatus is prepared to answer queries, we can |
| 1338 # announce the new build to all our watchers |
| 1339 @@ -620,7 +628,7 @@ |
| 1340 # Collect build numbers. |
| 1341 # Important: Only grab the *cached* builds numbers to reduce I/O. |
| 1342 current_builds = [b.getNumber() for b in self.currentBuilds] |
| 1343 - cached_builds = list(set(self.buildCache.keys() + current_builds)) |
| 1344 + cached_builds = list(set(self.buildCache.cache.keys() + current_builds)
) |
| 1345 cached_builds.sort() |
| 1346 result['cachedBuilds'] = cached_builds |
| 1347 result['currentBuilds'] = current_builds |
OLD | NEW |