Add datastore dumper tool.

service/datastore/dumper/dumper.go

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package dumper implements a very VERY dumb datastore-dumping debugging aid.
+// You shouldn't plan on having this work with the production datastore with any
+// appreciable amount of data.
+//
+// This will take an arbitrary query (or even a query for every entity in the
+// entire datastore), and print every entity to some output stream.
+package dumper
+
+import (
+        "fmt"
+        "io"
+        "os"
+        "sort"
+        "strings"
+
+        "github.com/luci/gae/service/datastore"
+        "golang.org/x/net/context"
+)
+
+// Key is a key into a PropFilterMap
+type Key struct {
+        Kind     string
+        PropName string
+}
+
+// A PropFilterMap maps from Kind+PropertyName tuples to a formatting function. You
+// may use this to specially format particular properties.
+type PropFilterMap map[Key]func(datastore.Property) string
+
+// KindFilterMap maps from a Kind to a formatting function. You may use this to
+// specially format particular Kinds. If this function returns an empty string,
+// the default formatting function (including any PropFilterMap entries) will be
+// used.
+type KindFilterMap map[string]func(*datastore.Key, datastore.PropertyMap) string
+
+// Config is a configured dumper.
+type Config struct {
+        // OutStream is the output stream to use. If this is nil, os.Stdout will be
+        // used.
+        OutStream io.Writer
+
+        // WithSpecial, if true, includes entities which have kinds that begin and
+        // end with "__". By default, these entities are skipped.
+        WithSpecial bool
+
+        // PropFilters is an optional property filter map for controlling the
+        // rendering of certain Kind/Property values.
+        PropFilters PropFilterMap
+
+        // KindFilters is an optional kind filter for controlling the rendering of
+        // certain Kind values.
+        KindFilters KindFilterMap
+}
+
+// Query will dump everything matching the provided query.
+//
+// If the provided query is nil, a kindless query without any filters will be
+// used.
+func (cfg Config) Query(c context.Context, q *datastore.Query) (n int, err error) {
+        ds := datastore.Get(c)
+
+        if q == nil {
+                q = datastore.NewQuery("")
+        }
+
+        out := cfg.OutStream
+        if out == nil {
+                out = os.Stdout
+        }
+
+        fmtVal := func(kind, name string, prop datastore.Property) string {
+                if fn := cfg.PropFilters[Key{kind, name}]; fn != nil {
+                        return fn(prop)
+                }
+                return prop.String()
+        }
+
+        prnt := func(format string, args ...interface{}) (err error) {
+                var amt int
+                amt, err = fmt.Fprintf(out, format, args...)
+                n += amt
+                return
+        }
+
+        prop := func(kind, name string, vals []datastore.Property) (err error) {
+                if len(vals) <= 1 {
+                        return prnt("  %q: [%s]\n", name, fmtVal(kind, name, vals[0]))
+                }
+                if err = prnt("  %q: [\n    %s", name, fmtVal(kind, name, vals[0])); err != nil {
+                        return
+                }
+                for _, v := range vals[1:] {
+                        if err = prnt(",\n    %s", fmtVal(kind, name, v)); err != nil {
+                                return
+                        }
+                }
+                return prnt("\n  ]\n")
+        }
+
+        err = ds.Run(q, func(pm datastore.PropertyMap) (err error) {
+                key := datastore.GetMetaDefault(pm, "key", nil).(*datastore.Key)

[dnj, 2016/06/06 14:01:38]

nit: I think "err" and "err" are a bit awkward here. I don't see much that is
gained from named output values on the inner function, so I'd prefer you just
"return err" in here.

Don't want to block CL on it though, so up to you.

+                if !cfg.WithSpecial && strings.HasPrefix(key.Kind(), "__") && strings.HasSuffix(key.Kind(), "__") {
+                        return
+                }
+                if err = prnt("\n%s:\n", key); err != nil {
+                        return
+                }
+                pm, _ = pm.Save(false)
+
+                // See if we have a KindFilter for this
+                if flt, ok := cfg.KindFilters[key.Kind()]; ok {
+                        if kindOut := flt(key, pm); kindOut != "" {
+                                for _, l := range strings.Split(kindOut, "\n") {
+                                        if err = prnt("  %s\n", l); err != nil {
+                                                return
+                                        }
+                                }
+                                return
+                        }
+                }
+
+                keys := make([]string, 0, len(pm))
+                for k := range pm {
+                        keys = append(keys, k)
+                }
+                sort.Strings(keys)
+                for _, k := range keys {
+                        if err = prop(key.Kind(), k, pm[k]); err != nil {
+                                return
+                        }
+                }
+                return
+        })
+        return
+}
+
+// Query dumps the provided query to stdout without special entities and with
+// default rendering.
+func Query(c context.Context, q *datastore.Query) {
+        Config{}.Query(c, q)
+}
+
+// All dumps all entities to stdout without special entities and with default
+// rendering.
+func All(c context.Context) {
+        Config{}.Query(c, nil)
+}