Add datastore iterating query filter.

filter/dsQueryBatch/filter_test.go

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package dsQueryBatch
+
+import (
+        "fmt"
+        "testing"
+
+        "github.com/luci/gae/filter/count"
+        "github.com/luci/gae/impl/memory"
+        ds "github.com/luci/gae/service/datastore"
+        "golang.org/x/net/context"
+
+        . "github.com/smartystreets/goconvey/convey"
+)
+
+type Item struct {
+        ID int64 `gae:"$id"`
+}
+
+func TestRun(t *testing.T) {
+        t.Parallel()
+
+        Convey("A memory with a counting filter and data set installed", t, func() {
+                c, cf := count.FilterRDS(memory.Use(context.Background()))
+
+                items := make([]*Item, 1024)
+                for i := range items {
+                        items[i] = &Item{int64(i + 1)}
+                }
+                if err := ds.Get(c).PutMulti(items); err != nil {
+                        panic(err)
+                }
+                ds.Get(c).Testable().CatchupIndexes()
+
+                for _, size := range []int{
+                        1,
+                        16,
+                        1024,
+                        2048,
+                        4096,
+                } {
+                        Convey(fmt.Sprintf(`With a batch filter size %d installed`, size), func() {
+                                c = BatchQueries(c, int32(size))
+                                q := ds.NewQuery("Item")
+
+                                Convey(`Can retrieve all of the items.`, func() {
+                                        var got []*Item
+                                        So(ds.Get(c).GetAll(q, &got), ShouldBeNil)
+                                        So(got, ShouldResemble, items)
+
+                                        // One call for every sub-query, plus one to hit Stop.
+                                        runCalls := (len(items) / size) + 1
+                                        So(cf.Run.Successes(), ShouldEqual, runCalls)
+                                })
+
+                                Convey(`With a limit of 128, will retrieve 128 items.`, func() {

[iannucci, 2016/03/31 22:40:31]

other tests?
batch 128 limit 127/129
non-even batch size?

[dnj, 2016/03/31 23:54:06]

Done.

+                                        const limit = 128
+                                        q = q.Limit(int32(limit))
+
+                                        var got []*Item
+                                        So(ds.Get(c).GetAll(q, &got), ShouldBeNil)
+                                        So(got, ShouldResemble, items[:limit])
+
+                                        // One call for every sub-query, plus one to hit Stop.
+                                        runCalls := (limit / size)
+                                        if size > limit {
+                                                runCalls++
+                                        }
+                                        So(cf.Run.Successes(), ShouldEqual, runCalls)
+                                })
+                        })
+                }
+        })
+}