Milo: Pubsub - Trim out pending build states if there are more than 25 per builder

milo/appengine/buildbot/pubsub.go

 // Copyright 2016 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.
 
 package buildbot
 
 import (
         "bytes"
         "compress/gzip"
         "compress/zlib"
@@ skipping 57 matching lines @@
         // Internal
         Internal bool
         // Data is the json serialzed and gzipped blob of the master data.
         Data []byte `gae:",noindex"`
         // Modified is when this entry was last modified.
         Modified time.Time
 }
 
 func putDSMasterJSON(
         c context.Context, master *buildbotMaster, internal bool) error {
+        // Trim pending build states.  These things are large and we can't really

[estaab, 2016/10/19 22:09:35]

How can we inform consumers that this was trimmed? We really need to do that.

[hinoka, 2016/10/19 22:23:11]

We send both the total number of pending builds (int) and details about the
first 25, so it would show up as something like

Pending Builds (100):
* detail...
* detail...
...23 more...

+        // store more than 25 of them.  If this becomes an issue again, we'll have
+        // to trim out things from the pending build state such as the changed
+        // file list, commit comments, etc.
+        for _, builder := range master.Builders {
+                if len(builder.PendingBuildStates) > 25 {
+                        builder.PendingBuildStates = builder.PendingBuildStates[0:25]
+                }
+        }
         entry := buildbotMasterEntry{
                 Name:     master.Name,
                 Internal: internal,
                 Modified: clock.Now(c).UTC(),
         }
         gzbs := bytes.Buffer{}
         gsw := gzip.NewWriter(&gzbs)
         cw := iotools.CountingWriter{Writer: gsw}
         e := json.NewEncoder(&cw)
         if err := e.Encode(master); err != nil {
@@ skipping 206 matching lines @@
                 if err != nil {
                         logging.WithError(err).Errorf(
                                 c, "Could not save master in datastore %s", err)
                         // This is transient, we do want PubSub to retry.
                         h.WriteHeader(500)
                         return
                 }
         }
         h.WriteHeader(200)
 }