OLD | NEW |
| (Empty) |
1 // Copyright 2015 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 package rawdatastore | |
6 | |
7 import ( | |
8 "bytes" | |
9 "errors" | |
10 "fmt" | |
11 "sort" | |
12 "time" | |
13 | |
14 "github.com/luci/gae/service/blobstore" | |
15 "github.com/luci/luci-go/common/cmpbin" | |
16 ) | |
17 | |
18 // WritePropertyMapDeterministic allows tests to make WritePropertyMap | |
19 // deterministic. | |
20 var WritePropertyMapDeterministic = false | |
21 | |
22 // ReadPropertyMapReasonableLimit sets a limit on the number of rows and | |
23 // number of properties per row which can be read by ReadPropertyMap. The | |
24 // total number of Property objects readable by this method is this number | |
25 // squared (e.g. Limit rows * Limit properties) | |
26 const ReadPropertyMapReasonableLimit uint64 = 30000 | |
27 | |
28 // ReadKeyNumToksReasonableLimit is the maximum number of Key tokens that | |
29 // ReadKey is willing to read for a single key. | |
30 const ReadKeyNumToksReasonableLimit uint64 = 50 | |
31 | |
32 // KeyContext controls whether the various Write and Read serializtion | |
33 // routines should encode the context of Keys (read: the appid and namespace). | |
34 // Frequently the appid and namespace of keys are known in advance and so there'
s | |
35 // no reason to redundantly encode them. | |
36 type KeyContext bool | |
37 | |
38 // With- and WithoutContext indicate if the serialization method should include | |
39 // context for Keys. See KeyContext for more information. | |
40 const ( | |
41 WithContext KeyContext = true | |
42 WithoutContext = false | |
43 ) | |
44 | |
45 // WriteKey encodes a key to the buffer. If context is WithContext, then this | |
46 // encoded value will include the appid and namespace of the key. | |
47 func WriteKey(buf Buffer, context KeyContext, k Key) (err error) { | |
48 // [appid ++ namespace]? ++ #tokens ++ tokens* | |
49 defer recoverTo(&err) | |
50 appid, namespace, toks := KeySplit(k) | |
51 if context == WithContext { | |
52 panicIf(buf.WriteByte(1)) | |
53 _, e := cmpbin.WriteString(buf, appid) | |
54 panicIf(e) | |
55 _, e = cmpbin.WriteString(buf, namespace) | |
56 panicIf(e) | |
57 } else { | |
58 panicIf(buf.WriteByte(0)) | |
59 } | |
60 _, e := cmpbin.WriteUint(buf, uint64(len(toks))) | |
61 panicIf(e) | |
62 for _, tok := range toks { | |
63 panicIf(WriteKeyTok(buf, tok)) | |
64 } | |
65 return nil | |
66 } | |
67 | |
68 // ReadKey deserializes a key from the buffer. The value of context must match | |
69 // the value of context that was passed to WriteKey when the key was encoded. | |
70 // If context == WithoutContext, then the appid and namespace parameters are | |
71 // used in the decoded Key. Otherwise they're ignored. | |
72 func ReadKey(buf Buffer, context KeyContext, appid, namespace string) (ret Key,
err error) { | |
73 defer recoverTo(&err) | |
74 actualCtx, e := buf.ReadByte() | |
75 panicIf(e) | |
76 | |
77 actualAid, actualNS := "", "" | |
78 if actualCtx == 1 { | |
79 actualAid, _, e = cmpbin.ReadString(buf) | |
80 panicIf(e) | |
81 actualNS, _, e = cmpbin.ReadString(buf) | |
82 panicIf(e) | |
83 } else if actualCtx != 0 { | |
84 err = fmt.Errorf("helper: expected actualCtx to be 0 or 1, got %
d", actualCtx) | |
85 return | |
86 } | |
87 | |
88 if context == WithoutContext { | |
89 // overrwrite with the supplied ones | |
90 actualAid = appid | |
91 actualNS = namespace | |
92 } | |
93 | |
94 numToks, _, e := cmpbin.ReadUint(buf) | |
95 panicIf(e) | |
96 if numToks > ReadKeyNumToksReasonableLimit { | |
97 err = fmt.Errorf("helper: tried to decode huge key of length %d"
, numToks) | |
98 return | |
99 } | |
100 | |
101 toks := make([]KeyTok, numToks) | |
102 for i := uint64(0); i < numToks; i++ { | |
103 toks[i], e = ReadKeyTok(buf) | |
104 panicIf(e) | |
105 } | |
106 | |
107 return NewKeyToks(actualAid, actualNS, toks), nil | |
108 } | |
109 | |
110 // WriteKeyTok writes a KeyTok to the buffer. You usually want WriteKey | |
111 // instead of this. | |
112 func WriteKeyTok(buf Buffer, tok KeyTok) (err error) { | |
113 // tok.kind ++ typ ++ [tok.stringID || tok.intID] | |
114 defer recoverTo(&err) | |
115 _, e := cmpbin.WriteString(buf, tok.Kind) | |
116 panicIf(e) | |
117 if tok.StringID != "" { | |
118 panicIf(buf.WriteByte(byte(PTString))) | |
119 _, e := cmpbin.WriteString(buf, tok.StringID) | |
120 panicIf(e) | |
121 } else { | |
122 panicIf(buf.WriteByte(byte(PTInt))) | |
123 _, e := cmpbin.WriteInt(buf, tok.IntID) | |
124 panicIf(e) | |
125 } | |
126 return nil | |
127 } | |
128 | |
129 // ReadKeyTok reads a KeyTok from the buffer. You usually want ReadKey | |
130 // instead of this. | |
131 func ReadKeyTok(buf Buffer) (ret KeyTok, err error) { | |
132 defer recoverTo(&err) | |
133 e := error(nil) | |
134 ret.Kind, _, e = cmpbin.ReadString(buf) | |
135 panicIf(e) | |
136 | |
137 typ, e := buf.ReadByte() | |
138 panicIf(e) | |
139 | |
140 switch PropertyType(typ) { | |
141 case PTString: | |
142 ret.StringID, _, err = cmpbin.ReadString(buf) | |
143 case PTInt: | |
144 ret.IntID, _, err = cmpbin.ReadInt(buf) | |
145 if err == nil && ret.IntID <= 0 { | |
146 err = errors.New("helper: decoded key with empty stringI
D and zero/negative intID") | |
147 } | |
148 default: | |
149 err = fmt.Errorf("helper: invalid type %s", PropertyType(typ)) | |
150 } | |
151 return | |
152 } | |
153 | |
154 // Write writes a GeoPoint to the buffer. | |
155 func (gp GeoPoint) Write(buf Buffer) (err error) { | |
156 defer recoverTo(&err) | |
157 _, e := cmpbin.WriteFloat64(buf, gp.Lat) | |
158 panicIf(e) | |
159 _, e = cmpbin.WriteFloat64(buf, gp.Lng) | |
160 return e | |
161 } | |
162 | |
163 // Read reads a GeoPoint from the buffer. | |
164 func (gp *GeoPoint) Read(buf Buffer) (err error) { | |
165 defer recoverTo(&err) | |
166 e := error(nil) | |
167 gp.Lat, _, e = cmpbin.ReadFloat64(buf) | |
168 panicIf(e) | |
169 | |
170 gp.Lng, _, e = cmpbin.ReadFloat64(buf) | |
171 panicIf(e) | |
172 | |
173 if !gp.Valid() { | |
174 err = fmt.Errorf("helper: decoded invalid GeoPoint: %v", gp) | |
175 } | |
176 return | |
177 } | |
178 | |
179 // WriteTime writes a time.Time in a byte-sortable way. | |
180 // | |
181 // This method truncates the time to microseconds and drops the timezone, | |
182 // because that's the (undocumented) way that the appengine SDK does it. | |
183 func WriteTime(buf Buffer, t time.Time) error { | |
184 name, off := t.Zone() | |
185 if name != "UTC" || off != 0 { | |
186 panic(fmt.Errorf("helper: UTC OR DEATH: %s", t)) | |
187 } | |
188 _, err := cmpbin.WriteUint(buf, uint64(t.Unix())*1e6+uint64(t.Nanosecond
()/1e3)) | |
189 return err | |
190 } | |
191 | |
192 // ReadTime reads a time.Time from the buffer. | |
193 func ReadTime(buf Buffer) (time.Time, error) { | |
194 v, _, err := cmpbin.ReadUint(buf) | |
195 if err != nil { | |
196 return time.Time{}, err | |
197 } | |
198 return time.Unix(int64(v/1e6), int64((v%1e6)*1e3)).UTC(), nil | |
199 } | |
200 | |
201 // Write writes a Property to the buffer. `context` behaves the same | |
202 // way that it does for WriteKey, but only has an effect if `p` contains a | |
203 // Key as its Value. | |
204 func (p *Property) Write(buf Buffer, context KeyContext) (err error) { | |
205 defer recoverTo(&err) | |
206 typb := byte(p.Type()) | |
207 if p.IndexSetting() == NoIndex { | |
208 typb |= 0x80 | |
209 } | |
210 panicIf(buf.WriteByte(typb)) | |
211 switch p.Type() { | |
212 case PTNull, PTBoolTrue, PTBoolFalse: | |
213 case PTInt: | |
214 _, err = cmpbin.WriteInt(buf, p.Value().(int64)) | |
215 case PTFloat: | |
216 _, err = cmpbin.WriteFloat64(buf, p.Value().(float64)) | |
217 case PTString: | |
218 _, err = cmpbin.WriteString(buf, p.Value().(string)) | |
219 case PTBytes: | |
220 if p.IndexSetting() == NoIndex { | |
221 _, err = cmpbin.WriteBytes(buf, p.Value().([]byte)) | |
222 } else { | |
223 _, err = cmpbin.WriteBytes(buf, p.Value().(ByteString)) | |
224 } | |
225 case PTTime: | |
226 err = WriteTime(buf, p.Value().(time.Time)) | |
227 case PTGeoPoint: | |
228 err = p.Value().(GeoPoint).Write(buf) | |
229 case PTKey: | |
230 err = WriteKey(buf, context, p.Value().(Key)) | |
231 case PTBlobKey: | |
232 _, err = cmpbin.WriteString(buf, string(p.Value().(blobstore.Key
))) | |
233 } | |
234 return | |
235 } | |
236 | |
237 // Read reads a Property from the buffer. `context`, `appid`, and | |
238 // `namespace` behave the same way they do for ReadKey, but only have an | |
239 // effect if the decoded property has a Key value. | |
240 func (p *Property) Read(buf Buffer, context KeyContext, appid, namespace string)
(err error) { | |
241 val := interface{}(nil) | |
242 typb, err := buf.ReadByte() | |
243 if err != nil { | |
244 return | |
245 } | |
246 is := ShouldIndex | |
247 if (typb & 0x80) != 0 { | |
248 is = NoIndex | |
249 } | |
250 switch PropertyType(typb & 0x7f) { | |
251 case PTNull: | |
252 case PTBoolTrue: | |
253 val = true | |
254 case PTBoolFalse: | |
255 val = false | |
256 case PTInt: | |
257 val, _, err = cmpbin.ReadInt(buf) | |
258 case PTFloat: | |
259 val, _, err = cmpbin.ReadFloat64(buf) | |
260 case PTString: | |
261 val, _, err = cmpbin.ReadString(buf) | |
262 case PTBytes: | |
263 b := []byte(nil) | |
264 if b, _, err = cmpbin.ReadBytes(buf); err != nil { | |
265 break | |
266 } | |
267 if is == NoIndex { | |
268 val = b | |
269 } else { | |
270 val = ByteString(b) | |
271 } | |
272 case PTTime: | |
273 val, err = ReadTime(buf) | |
274 case PTGeoPoint: | |
275 gp := GeoPoint{} | |
276 err = gp.Read(buf) | |
277 val = gp | |
278 case PTKey: | |
279 val, err = ReadKey(buf, context, appid, namespace) | |
280 case PTBlobKey: | |
281 s := "" | |
282 if s, _, err = cmpbin.ReadString(buf); err != nil { | |
283 break | |
284 } | |
285 val = blobstore.Key(s) | |
286 default: | |
287 err = fmt.Errorf("read: unknown type! %v", typb) | |
288 } | |
289 if err == nil { | |
290 err = p.SetValue(val, is) | |
291 } | |
292 return | |
293 } | |
294 | |
295 // Write writes an entire PropertyMap to the buffer. `context` | |
296 // behaves the same way that it does for WriteKey. If | |
297 // WritePropertyMapDeterministic is true, then the rows will be sorted by | |
298 // property name before they're serialized to buf (mostly useful for testing, | |
299 // but also potentially useful if you need to make a hash of the property data). | |
300 func (pm PropertyMap) Write(buf Buffer, context KeyContext) (err error) { | |
301 defer recoverTo(&err) | |
302 rows := make(sort.StringSlice, 0, len(pm)) | |
303 tmpBuf := &bytes.Buffer{} | |
304 for name, vals := range pm { | |
305 tmpBuf.Reset() | |
306 _, e := cmpbin.WriteString(tmpBuf, name) | |
307 panicIf(e) | |
308 _, e = cmpbin.WriteUint(tmpBuf, uint64(len(vals))) | |
309 panicIf(e) | |
310 for _, p := range vals { | |
311 panicIf(p.Write(tmpBuf, context)) | |
312 } | |
313 rows = append(rows, tmpBuf.String()) | |
314 } | |
315 | |
316 if WritePropertyMapDeterministic { | |
317 rows.Sort() | |
318 } | |
319 | |
320 _, e := cmpbin.WriteUint(buf, uint64(len(pm))) | |
321 panicIf(e) | |
322 for _, r := range rows { | |
323 _, e := buf.WriteString(r) | |
324 panicIf(e) | |
325 } | |
326 return | |
327 } | |
328 | |
329 // Read reads a PropertyMap from the buffer. `context` and | |
330 // friends behave the same way that they do for ReadKey. | |
331 func (pm PropertyMap) Read(buf Buffer, context KeyContext, appid, namespace stri
ng) (err error) { | |
332 defer recoverTo(&err) | |
333 | |
334 numRows := uint64(0) | |
335 numRows, _, e := cmpbin.ReadUint(buf) | |
336 panicIf(e) | |
337 if numRows > ReadPropertyMapReasonableLimit { | |
338 err = fmt.Errorf("helper: tried to decode map with huge number o
f rows %d", numRows) | |
339 return | |
340 } | |
341 | |
342 name, prop := "", Property{} | |
343 for i := uint64(0); i < numRows; i++ { | |
344 name, _, e = cmpbin.ReadString(buf) | |
345 panicIf(e) | |
346 | |
347 numProps, _, e := cmpbin.ReadUint(buf) | |
348 panicIf(e) | |
349 if numProps > ReadPropertyMapReasonableLimit { | |
350 err = fmt.Errorf("helper: tried to decode map with huge
number of properties %d", numProps) | |
351 return | |
352 } | |
353 props := make([]Property, 0, numProps) | |
354 for j := uint64(0); j < numProps; j++ { | |
355 panicIf(prop.Read(buf, context, appid, namespace)) | |
356 props = append(props, prop) | |
357 } | |
358 pm[name] = props | |
359 } | |
360 return | |
361 } | |
362 | |
363 type parseError error | |
364 | |
365 func panicIf(err error) { | |
366 if err != nil { | |
367 panic(parseError(err)) | |
368 } | |
369 } | |
370 | |
371 func recoverTo(err *error) { | |
372 if r := recover(); r != nil { | |
373 if rerr := r.(parseError); rerr != nil { | |
374 *err = error(rerr) | |
375 } | |
376 } | |
377 } | |
OLD | NEW |