impl/memory: Implement Queries

impl/memory/datastore_query_execution.go

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package memory
+
+import (
+        "bytes"
+        "fmt"
+
+        ds "github.com/luci/gae/service/datastore"
+        "github.com/luci/gae/service/datastore/serialize"
+        "github.com/luci/luci-go/common/cmpbin"
+)
+
+type queryStrategy interface {
+        // handle applies the strategy to the embedded user callback.
+        //   - rawData is the slice of encoded Properties from the index row
+        //     (correctly de-inverted).
+        //   - data is the slice of decoded Properties from the index row
+        //   - key is the decoded Key from the index row (the last item in rawData and
+        //     data)
+        //   - gc is the getCursor function to be passed to the user's callback
+        handle(rawData [][]byte, data []ds.Property, key ds.Key, gc func() (ds.Cursor, error)) bool
+}
+
+type projectionLookup struct {
+        suffixIndex  int
+        propertyName string
+}
+
+type projectionStrategy struct {
+        cb ds.RawRunCB
+
+        project  []projectionLookup
+        distinct map[string]struct{}
+}
+
+func newProjectionStrategy(q *queryImpl, rq *reducedQuery, cb ds.RawRunCB) queryStrategy {
+        projectionLookups := make([]projectionLookup, len(q.project))
+        for i, prop := range q.project {
+                projectionLookups[i].propertyName = prop
+                foundIt := false
+                for j, col := range rq.suffixFormat {
+                        if col.Property == prop {
+                                projectionLookups[i].suffixIndex = j
+                                foundIt = true
+                                break
+                        }
+                }
+                if !foundIt {
+                        impossible(fmt.Errorf("planning a strategy for an unfulfillable query?"))
+                }
+        }
+        ret := &projectionStrategy{cb: cb, project: projectionLookups}
+        if q.distinct {
+                ret.distinct = map[string]struct{}{}
+        }
+        return ret
+}
+
+func (s *projectionStrategy) handle(rawData [][]byte, suffixData []ds.Property, key ds.Key, gc func() (ds.Cursor, error)) bool {
+        projectedRaw := [][]byte(nil)
+        if s.distinct != nil {
+                projectedRaw = make([][]byte, len(suffixData))
+        }
+        pmap := make(ds.PropertyMap, len(s.project))
+        for i, p := range s.project {
+                if s.distinct != nil {
+                        projectedRaw[i] = rawData[p.suffixIndex]
+                }
+                pmap[p.propertyName] = []ds.Property{suffixData[p.suffixIndex]}
+        }
+        if s.distinct != nil {
+                rawString := string(bjoin(projectedRaw...))
+                if _, ok := s.distinct[rawString]; ok {
+                        return true
+                }
+                s.distinct[rawString] = struct{}{}
+        }
+        return s.cb(key, pmap, gc)
+}
+
+type keysOnlyStrategy ds.RawRunCB
+
+func (s keysOnlyStrategy) handle(_ [][]byte, _ []ds.Property, key ds.Key, gc func() (ds.Cursor, error)) bool {
+        return ds.RawRunCB(s)(key, nil, gc)
+}
+
+type normalStrategy struct {
+        cb ds.RawRunCB
+
+        ns   string
+        head *memCollection
+}
+
+func newNormalStrategy(ns string, cb ds.RawRunCB, head *memStore) queryStrategy {
+        coll := head.GetCollection("ents:" + ns)
+        if coll == nil {
+                return nil
+        }
+        return &normalStrategy{cb, ns, coll}
+}
+
+func (s *normalStrategy) handle(rawData [][]byte, _ []ds.Property, key ds.Key, gc func() (ds.Cursor, error)) bool {
+        rawEnt := s.head.Get(rawData[len(rawData)-1][1:])
+        if rawEnt == nil {
+                // entity doesn't exist at head
+                return true
+        }
+        pm, err := serialize.ReadPropertyMap(bytes.NewBuffer(rawEnt), serialize.WithoutContext, globalAppID, s.ns)
+        memoryCorruption(err)
+
+        return s.cb(key, pm, gc)
+}
+
+func pickQueryStrategy(q *queryImpl, rq *reducedQuery, cb ds.RawRunCB, head *memStore) queryStrategy {
+        if q.keysOnly {
+                return keysOnlyStrategy(cb)
+        } else if len(q.project) > 0 {
+                return newProjectionStrategy(q, rq, cb)

[dnj (Google), 2015/08/23 06:50:06]

No need for "else" here.

[iannucci, 2015/08/23 18:19:43]

Done.

+        }
+        return newNormalStrategy(rq.ns, cb, head)
+}
+
+func parseSuffix(ns string, suffixFormat []ds.IndexColumn, suffix []byte) (raw [][]byte, decoded []ds.Property) {
+        buf := serialize.Invertible(bytes.NewBuffer(suffix))
+        decoded = make([]ds.Property, len(suffixFormat))
+        raw = make([][]byte, len(suffixFormat))
+
+        err := error(nil)
+        prevOffset := 0
+        for i := range decoded {
+                needInvert := suffixFormat[i].Direction == ds.DESCENDING
+
+                buf.SetInvert(needInvert)

[dnj (Google), 2015/08/23 06:50:06]

Just inline the condition here: buf.SetInvert(suffixFormat[i].Direction ==
ds.DESCENDING)

[iannucci, 2015/08/23 18:19:42]

Need this below (line 141) as well

+                decoded[i], err = serialize.ReadProperty(buf, serialize.WithoutContext, globalAppID, ns)
+                memoryCorruption(err)
+
+                offset := len(suffix) - buf.Len()
+                if needInvert {
+                        raw[i] = invert(suffix[prevOffset:offset])

[dnj (Google), 2015/08/23 06:50:06]

Consider walking suffix:
suffix = suffix[:offset]

IMO would make this a little cleaner, no need for prevOffset, etc.

[iannucci, 2015/08/23 18:19:42]

ya. done

+                } else {
+                        raw[i] = suffix[prevOffset:offset]
+                }
+                prevOffset = offset
+        }
+
+        return
+}
+
+func executeQuery(origQ ds.Query, ns string, isTxn bool, idx, head *memStore, cb ds.RawRunCB) error {
+        q, ok := origQ.(*queryImpl)
+        if !ok {
+                return fmt.Errorf("unsupported Query implementation: %#v", q)

[dnj (Google), 2015/08/23 06:50:06]

%T?

[iannucci, 2015/08/23 18:19:43]

Done.

+        }
+
+        rq, err := q.reduce(ns, isTxn)
+        if err != nil {
+                if err == errQueryDone {
+                        return nil
+                }
+                return err
+        }
+
+        strategy := pickQueryStrategy(q, rq, cb, head)
+        if strategy == nil {
+                // e.g. the normalStrategy found that there were NO entities in the current
+                // namespace.
+                return nil
+        }
+
+        dedup := map[string]struct{}{}
+
+        offset := q.offset
+        limit := q.limit
+        hasLimit := limit < 0

[dnj (Google), 2015/08/23 06:50:06]

There's a limit if either (a) limit < 0, or (b) limit > 0; therefore, "hasLimit"
should be "limit != 0", no?

[iannucci, 2015/08/23 18:19:43]

:)

"https://cloud.google.com/appengine/docs/go/datastore/reference#Query.Limit"

See my facepalm on (*queryImpl).Limit

+
+        cursorPrefix := []byte(nil)
+        getCursorFn := func(suffix []byte) func() (ds.Cursor, error) {
+                return func() (ds.Cursor, error) {
+                        if cursorPrefix == nil {
+                                buf := &bytes.Buffer{}
+                                _, err := cmpbin.WriteUint(buf, uint64(len(rq.suffixFormat)))
+                                memoryCorruption(err)
+
+                                for _, col := range rq.suffixFormat {
+                                        err := serialize.WriteIndexColumn(buf, col)
+                                        memoryCorruption(err)
+                                }
+                                cursorPrefix = buf.Bytes()
+                        }
+                        return queryCursor(bjoin(cursorPrefix, suffix)), nil
+                }
+        }
+
+        idxs, err := getIndicies(rq, idx)
+        if err == errQueryDone {
+                return nil
+        } else if err != nil {

[dnj (Google), 2015/08/23 06:50:06]

No need for "else".

[iannucci, 2015/08/23 18:19:42]

Done.

+                return err
+        }
+
+        multiIterate(idxs, func(suffix []byte) bool {
+                if offset > 0 {
+                        offset--
+                        return true
+                }
+                if hasLimit {
+                        if limit <= 0 {
+                                return false
+                        }
+                        limit--
+                }
+
+                rawData, data := parseSuffix(ns, rq.suffixFormat, suffix)

[dnj (Google), 2015/08/23 06:50:06]

I don't think "data" is a good name here. Maybe "decodedProps".

[iannucci, 2015/08/23 18:19:42]

Done.

+
+                keyProp := data[len(data)-1]
+                if keyProp.Type() != ds.PTKey {
+                        impossible(fmt.Errorf("decoded index row doesn't end with a Key: %#v", data))

[dnj (Google), 2015/08/23 06:50:06]

%T?

[iannucci, 2015/08/23 18:19:43]

Nah. Changed to keyProp tho.

+                }
+                keyRaw := string(rawData[len(rawData)-1])
+                if _, ok := dedup[keyRaw]; ok {
+                        // dedup'd!
+                        return true
+                }
+                dedup[keyRaw] = struct{}{}
+
+                return strategy.handle(
+                        rawData, data, keyProp.Value().(ds.Key),
+                        getCursorFn(suffix))
+        })
+
+        return nil
+}