Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(324)

Side by Side Diff: impl/memory/datastore_query_execution.go

Issue 1312433013: Switch to external stringset implementation. (Closed) Base URL: https://github.com/luci/gae.git@master
Patch Set: fix nitish Created 5 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « impl/memory/datastore_query.go ('k') | impl/memory/stringset.go » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright 2015 The Chromium Authors. All rights reserved. 1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 package memory 5 package memory
6 6
7 import ( 7 import (
8 "bytes" 8 "bytes"
9 "fmt" 9 "fmt"
10 10
11 ds "github.com/luci/gae/service/datastore" 11 ds "github.com/luci/gae/service/datastore"
12 "github.com/luci/gae/service/datastore/serialize" 12 "github.com/luci/gae/service/datastore/serialize"
13 "github.com/luci/luci-go/common/cmpbin" 13 "github.com/luci/luci-go/common/cmpbin"
14 "github.com/luci/luci-go/common/stringset"
14 ) 15 )
15 16
16 type queryStrategy interface { 17 type queryStrategy interface {
17 // handle applies the strategy to the embedded user callback. 18 // handle applies the strategy to the embedded user callback.
18 // - rawData is the slice of encoded Properties from the index row 19 // - rawData is the slice of encoded Properties from the index row
19 // (correctly de-inverted). 20 // (correctly de-inverted).
20 // - decodedProps is the slice of decoded Properties from the index ro w 21 // - decodedProps is the slice of decoded Properties from the index ro w
21 // - key is the decoded Key from the index row (the last item in rawDa ta and 22 // - key is the decoded Key from the index row (the last item in rawDa ta and
22 // decodedProps) 23 // decodedProps)
23 // - gc is the getCursor function to be passed to the user's callback 24 // - gc is the getCursor function to be passed to the user's callback
24 handle(rawData [][]byte, decodedProps []ds.Property, key ds.Key, gc func () (ds.Cursor, error)) bool 25 handle(rawData [][]byte, decodedProps []ds.Property, key ds.Key, gc func () (ds.Cursor, error)) bool
25 } 26 }
26 27
27 type projectionLookup struct { 28 type projectionLookup struct {
28 suffixIndex int 29 suffixIndex int
29 propertyName string 30 propertyName string
30 } 31 }
31 32
32 type projectionStrategy struct { 33 type projectionStrategy struct {
33 cb ds.RawRunCB 34 cb ds.RawRunCB
34 35
35 project []projectionLookup 36 project []projectionLookup
36 » distinct stringSet 37 » distinct stringset.Set
37 } 38 }
38 39
39 func newProjectionStrategy(q *queryImpl, rq *reducedQuery, cb ds.RawRunCB) query Strategy { 40 func newProjectionStrategy(q *queryImpl, rq *reducedQuery, cb ds.RawRunCB) query Strategy {
40 projectionLookups := make([]projectionLookup, len(q.project)) 41 projectionLookups := make([]projectionLookup, len(q.project))
41 for i, prop := range q.project { 42 for i, prop := range q.project {
42 projectionLookups[i].propertyName = prop 43 projectionLookups[i].propertyName = prop
43 lookupErr := fmt.Errorf("planning a strategy for an unfulfillabl e query?") 44 lookupErr := fmt.Errorf("planning a strategy for an unfulfillabl e query?")
44 for j, col := range rq.suffixFormat { 45 for j, col := range rq.suffixFormat {
45 if col.Property == prop { 46 if col.Property == prop {
46 projectionLookups[i].suffixIndex = j 47 projectionLookups[i].suffixIndex = j
47 lookupErr = nil 48 lookupErr = nil
48 break 49 break
49 } 50 }
50 } 51 }
51 impossible(lookupErr) 52 impossible(lookupErr)
52 } 53 }
53 ret := &projectionStrategy{cb: cb, project: projectionLookups} 54 ret := &projectionStrategy{cb: cb, project: projectionLookups}
54 if q.distinct { 55 if q.distinct {
55 » » ret.distinct = stringSet{} 56 » » ret.distinct = stringset.New(0)
56 } 57 }
57 return ret 58 return ret
58 } 59 }
59 60
60 func (s *projectionStrategy) handle(rawData [][]byte, decodedProps []ds.Property , key ds.Key, gc func() (ds.Cursor, error)) bool { 61 func (s *projectionStrategy) handle(rawData [][]byte, decodedProps []ds.Property , key ds.Key, gc func() (ds.Cursor, error)) bool {
61 projectedRaw := [][]byte(nil) 62 projectedRaw := [][]byte(nil)
62 if s.distinct != nil { 63 if s.distinct != nil {
63 projectedRaw = make([][]byte, len(decodedProps)) 64 projectedRaw = make([][]byte, len(decodedProps))
64 } 65 }
65 pmap := make(ds.PropertyMap, len(s.project)) 66 pmap := make(ds.PropertyMap, len(s.project))
66 for i, p := range s.project { 67 for i, p := range s.project {
67 if s.distinct != nil { 68 if s.distinct != nil {
68 projectedRaw[i] = rawData[p.suffixIndex] 69 projectedRaw[i] = rawData[p.suffixIndex]
69 } 70 }
70 pmap[p.propertyName] = []ds.Property{decodedProps[p.suffixIndex] } 71 pmap[p.propertyName] = []ds.Property{decodedProps[p.suffixIndex] }
71 } 72 }
72 if s.distinct != nil { 73 if s.distinct != nil {
73 » » if !s.distinct.add(string(bjoin(projectedRaw...))) { 74 » » if !s.distinct.Add(string(bjoin(projectedRaw...))) {
74 return true 75 return true
75 } 76 }
76 } 77 }
77 return s.cb(key, pmap, gc) 78 return s.cb(key, pmap, gc)
78 } 79 }
79 80
80 type keysOnlyStrategy struct { 81 type keysOnlyStrategy struct {
81 cb ds.RawRunCB 82 cb ds.RawRunCB
82 83
83 » dedup stringSet 84 » dedup stringset.Set
84 } 85 }
85 86
86 func (s *keysOnlyStrategy) handle(rawData [][]byte, _ []ds.Property, key ds.Key, gc func() (ds.Cursor, error)) bool { 87 func (s *keysOnlyStrategy) handle(rawData [][]byte, _ []ds.Property, key ds.Key, gc func() (ds.Cursor, error)) bool {
87 » if !s.dedup.add(string(rawData[len(rawData)-1])) { 88 » if !s.dedup.Add(string(rawData[len(rawData)-1])) {
88 return true 89 return true
89 } 90 }
90 return s.cb(key, nil, gc) 91 return s.cb(key, nil, gc)
91 } 92 }
92 93
93 type normalStrategy struct { 94 type normalStrategy struct {
94 cb ds.RawRunCB 95 cb ds.RawRunCB
95 96
96 ns string 97 ns string
97 head *memCollection 98 head *memCollection
98 » dedup stringSet 99 » dedup stringset.Set
99 } 100 }
100 101
101 func newNormalStrategy(ns string, cb ds.RawRunCB, head *memStore) queryStrategy { 102 func newNormalStrategy(ns string, cb ds.RawRunCB, head *memStore) queryStrategy {
102 coll := head.GetCollection("ents:" + ns) 103 coll := head.GetCollection("ents:" + ns)
103 if coll == nil { 104 if coll == nil {
104 return nil 105 return nil
105 } 106 }
106 » return &normalStrategy{cb, ns, coll, stringSet{}} 107 » return &normalStrategy{cb, ns, coll, stringset.New(0)}
107 } 108 }
108 109
109 func (s *normalStrategy) handle(rawData [][]byte, _ []ds.Property, key ds.Key, g c func() (ds.Cursor, error)) bool { 110 func (s *normalStrategy) handle(rawData [][]byte, _ []ds.Property, key ds.Key, g c func() (ds.Cursor, error)) bool {
110 rawKey := rawData[len(rawData)-1] 111 rawKey := rawData[len(rawData)-1]
111 » if !s.dedup.add(string(rawKey)) { 112 » if !s.dedup.Add(string(rawKey)) {
112 return true 113 return true
113 } 114 }
114 115
115 rawEnt := s.head.Get(rawKey) 116 rawEnt := s.head.Get(rawKey)
116 if rawEnt == nil { 117 if rawEnt == nil {
117 // entity doesn't exist at head 118 // entity doesn't exist at head
118 return true 119 return true
119 } 120 }
120 pm, err := serialize.ReadPropertyMap(bytes.NewBuffer(rawEnt), serialize. WithoutContext, globalAppID, s.ns) 121 pm, err := serialize.ReadPropertyMap(bytes.NewBuffer(rawEnt), serialize. WithoutContext, globalAppID, s.ns)
121 memoryCorruption(err) 122 memoryCorruption(err)
122 123
123 return s.cb(key, pm, gc) 124 return s.cb(key, pm, gc)
124 } 125 }
125 126
126 func pickQueryStrategy(q *queryImpl, rq *reducedQuery, cb ds.RawRunCB, head *mem Store) queryStrategy { 127 func pickQueryStrategy(q *queryImpl, rq *reducedQuery, cb ds.RawRunCB, head *mem Store) queryStrategy {
127 if q.keysOnly { 128 if q.keysOnly {
128 » » return &keysOnlyStrategy{cb, stringSet{}} 129 » » return &keysOnlyStrategy{cb, stringset.New(0)}
129 } 130 }
130 if len(q.project) > 0 { 131 if len(q.project) > 0 {
131 return newProjectionStrategy(q, rq, cb) 132 return newProjectionStrategy(q, rq, cb)
132 } 133 }
133 return newNormalStrategy(rq.ns, cb, head) 134 return newNormalStrategy(rq.ns, cb, head)
134 } 135 }
135 136
136 func parseSuffix(ns string, suffixFormat []ds.IndexColumn, suffix []byte, count int) (raw [][]byte, decoded []ds.Property) { 137 func parseSuffix(ns string, suffixFormat []ds.IndexColumn, suffix []byte, count int) (raw [][]byte, decoded []ds.Property) {
137 buf := serialize.Invertible(bytes.NewBuffer(suffix)) 138 buf := serialize.Invertible(bytes.NewBuffer(suffix))
138 decoded = make([]ds.Property, len(suffixFormat)) 139 decoded = make([]ds.Property, len(suffixFormat))
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after
229 impossible(fmt.Errorf("decoded index row doesn't end wit h a Key: %#v", keyProp)) 230 impossible(fmt.Errorf("decoded index row doesn't end wit h a Key: %#v", keyProp))
230 } 231 }
231 232
232 return strategy.handle( 233 return strategy.handle(
233 rawData, decodedProps, keyProp.Value().(ds.Key), 234 rawData, decodedProps, keyProp.Value().(ds.Key),
234 getCursorFn(suffix)) 235 getCursorFn(suffix))
235 }) 236 })
236 237
237 return nil 238 return nil
238 } 239 }
OLDNEW
« no previous file with comments | « impl/memory/datastore_query.go ('k') | impl/memory/stringset.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698