datastore: variadic Get, Put, Exists, Delete.

service/datastore/multiarg.go

Index: service/datastore/multiarg.go
diff --git a/service/datastore/multiarg.go b/service/datastore/multiarg.go
index ab335c9ef634ba1ab4fa5817ecf3b649231a229f..a04f9a5adf3e01370b4205aaa358cc888a0cbbbc 100644
--- a/service/datastore/multiarg.go
+++ b/service/datastore/multiarg.go
@@ -20,155 +20,148 @@ type multiArgType struct {
 	newElem   func() reflect.Value
 }
 
-func (mat *multiArgType) GetKeysPMs(aid, ns string, slice reflect.Value, meta bool) ([]*Key, []PropertyMap, error) {
-	retKey := make([]*Key, slice.Len())
-	retPM := make([]PropertyMap, slice.Len())
-	getter := mat.getPM
-	if meta {
-		getter = func(slot reflect.Value) (PropertyMap, error) {
-			return mat.getMetaPM(slot), nil
-		}
-	}
-	lme := errors.NewLazyMultiError(len(retKey))
-	for i := range retKey {
-		key, err := mat.getKey(aid, ns, slice.Index(i))
-		if !lme.Assign(i, err) {
-			retKey[i] = key
-			pm, err := getter(slice.Index(i))
-			if !lme.Assign(i, err) {
-				retPM[i] = pm
-			}
-		}
-	}
-	return retKey, retPM, lme.Get()
-}
-
-// parseMultiArg checks that v has type []S, []*S, []I, []P or []*P, for some
+// parseArg checks that et is of type S, *S, I, P or *P, for some
 // struct type S, for some interface type I, or some non-interface non-pointer
 // type P such that P or *P implements PropertyLoadSaver.
-func parseMultiArg(e reflect.Type) multiArgType {
-	if e.Kind() != reflect.Slice {
-		panic(fmt.Errorf("invalid argument type: expected slice, got %s", e))
+//
+// If allowKeys is true, a read-only key extraction multiArgType will be
+// returned if et is a *Key.
+func parseArg(et reflect.Type, keysOnly bool) *multiArgType {

[iannucci, 2016/05/28 02:50:53]

s/keysOnly/allowKeys

[dnj, 2016/05/28 17:47:21]

Done.

+	if keysOnly && et == typeOfKey {
+		return multiArgTypeKeyExtraction()
 	}
-	return parseArg(e.Elem(), true)
-}
 
-// parseArg checks that et is of type S, *S, I, P or *P, for some
-// struct type S, for some interface type I, or some non-interface non-pointer
-// type P such that P or *P implements PropertyLoadSaver.
-func parseArg(et reflect.Type, multi bool) multiArgType {
-	if reflect.PtrTo(et).Implements(typeOfPropertyLoadSaver) {
+	if et.Kind() == reflect.Interface {
+		return multiArgTypeInterface()
+	}
+
+	// If a map type implements an interface, its pointer is also considered to
+	// implement that interface.
+	//
+	// In this case, we have special pointer-to-map logic in multiArgTypePLS.
+	if et.Implements(typeOfPropertyLoadSaver) {
 		return multiArgTypePLS(et)
 	}
-	if et.Implements(typeOfPropertyLoadSaver) && et.Kind() != reflect.Interface {
-		return multiArgTypePLSPtr(et.Elem())
+	if reflect.PtrTo(et).Implements(typeOfPropertyLoadSaver) {
+		return multiArgTypePLSPtr(et)
 	}
+
 	switch et.Kind() {
-	case reflect.Struct:
-		return multiArgTypeStruct(et)
-	case reflect.Interface:
-		return multiArgTypeInterface()
 	case reflect.Ptr:
-		et = et.Elem()
-		if et.Kind() == reflect.Struct {
+		if et.Elem().Kind() == reflect.Struct {
 			return multiArgTypeStructPtr(et)
 		}
+
+	case reflect.Struct:
+		return multiArgTypeStruct(et)
 	}
-	if multi {
-		panic(fmt.Errorf("invalid argument type: []%s", et))
+
+	return nil
+}
+
+// parseMultiArg checks that v has type []S, []*S, []I, []P or []*P, for some
+// struct type S, for some interface type I, or some non-interface non-pointer
+// type P such that P or *P implements PropertyLoadSaver.
+func mustParseMultiArg(et reflect.Type) *multiArgType {
+	if et.Kind() != reflect.Slice {
+		panic(fmt.Errorf("invalid argument type: expected slice, got %s", et))
 	}
-	panic(fmt.Errorf("invalid argument type: %s", et))
+	return mustParseArg(et.Elem())
 }
 
-type newKeyFunc func(kind, sid string, iid int64, par Key) Key
+func mustParseArg(et reflect.Type) *multiArgType {
+	if mat := parseArg(et, false); mat != nil {
+		return mat
+	}
+	panic(fmt.Errorf("invalid argument type: %s is not a PLS or pointer-to-struct", et))
+}
 
-// multiArgTypePLS == []P
-// *P implements PropertyLoadSaver
-func multiArgTypePLS(et reflect.Type) multiArgType {
+// multiArgTypePLS handles the case where et implements PropertyLoadSaver.
+//
+// This handles the special case of pointer-to-map (see parseArg).
+func multiArgTypePLS(et reflect.Type) *multiArgType {
 	ret := multiArgType{
 		getKey: func(aid, ns string, slot reflect.Value) (*Key, error) {
-			return newKeyObjErr(aid, ns, slot.Addr().Interface())
+			return newKeyObjErr(aid, ns, getMGS(slot.Interface()))
 		},
 		getPM: func(slot reflect.Value) (PropertyMap, error) {
-			return slot.Addr().Interface().(PropertyLoadSaver).Save(true)
+			return slot.Interface().(PropertyLoadSaver).Save(true)
 		},
 		getMetaPM: func(slot reflect.Value) PropertyMap {
-			return getMGS(slot.Addr().Interface()).GetAllMeta()
+			return getMGS(slot.Interface()).GetAllMeta()
 		},
 		setPM: func(slot reflect.Value, pm PropertyMap) error {
-			return slot.Addr().Interface().(PropertyLoadSaver).Load(pm)
+			return slot.Interface().(PropertyLoadSaver).Load(pm)
 		},
 		setKey: func(slot reflect.Value, k *Key) {
-			PopulateKey(slot.Addr().Interface(), k)
+			PopulateKey(slot.Interface(), k)
 		},
 	}
-	if et.Kind() == reflect.Map {
+	switch et.Kind() {
+	case reflect.Map:

[iannucci, 2016/05/28 02:50:53]

I think chans also can hit this.

[dnj, 2016/05/28 17:47:21]

Done. Unfortunately, there is *no way* to communicate the capacity of the
channel at the moment. I made a comment to this extent. I am reluctant to pour
time into that because neither of us can imagine a good use case for a channel
PLS.

 		ret.newElem = func() reflect.Value {
-			// Create a *map so that way slot.Addr() works above when this is
-			// called from Run(). Otherwise the map is 'unaddressable' according
-			// to reflect.  ¯\_(ツ)_/¯
-			ptr := reflect.New(et)
-			ptr.Elem().Set(reflect.MakeMap(et))
-			return ptr.Elem()
+			return reflect.MakeMap(et)
 		}
-	} else {
+
+	case reflect.Ptr:
+		mapElem := et.Elem()
+		if mapElem.Kind() == reflect.Map {
+			ret.newElem = func() reflect.Value {
+				ptr := reflect.New(mapElem)
+				ptr.Elem().Set(reflect.MakeMap(mapElem))
+				return ptr
+			}
+		}
+	}
+
+	if ret.newElem == nil {
 		ret.newElem = func() reflect.Value {
-			return reflect.New(et).Elem()
+			return reflect.New(et.Elem())
 		}
 	}
-	return ret
+	return &ret
 }
 
-// multiArgTypePLSPtr == []*P
-// *P implements PropertyLoadSaver
-func multiArgTypePLSPtr(et reflect.Type) multiArgType {
-	ret := multiArgType{
+// multiArgTypePLSPtr handles the case where et doesn't implement
+// PropertyLoadSaver, but a pointer to et does.
+func multiArgTypePLSPtr(et reflect.Type) *multiArgType {
+	return &multiArgType{
 		getKey: func(aid, ns string, slot reflect.Value) (*Key, error) {
-			return newKeyObjErr(aid, ns, slot.Interface())
+			return newKeyObjErr(aid, ns, getMGS(slot.Addr().Interface()))
 		},
 		getPM: func(slot reflect.Value) (PropertyMap, error) {
-			return slot.Interface().(PropertyLoadSaver).Save(true)
+			return slot.Addr().Interface().(PropertyLoadSaver).Save(true)
 		},
 		getMetaPM: func(slot reflect.Value) PropertyMap {
-			return getMGS(slot.Interface()).GetAllMeta()
+			return getMGS(slot.Addr().Interface()).GetAllMeta()
 		},
 		setPM: func(slot reflect.Value, pm PropertyMap) error {
-			return slot.Interface().(PropertyLoadSaver).Load(pm)
+			return slot.Addr().Interface().(PropertyLoadSaver).Load(pm)
 		},
 		setKey: func(slot reflect.Value, k *Key) {
-			PopulateKey(slot.Interface(), k)
+			PopulateKey(slot.Addr().Interface(), k)
+		},
+		newElem: func() reflect.Value {
+			return reflect.New(et).Elem()
 		},
 	}
-	if et.Kind() == reflect.Map {
-		ret.newElem = func() reflect.Value {
-			ptr := reflect.New(et)
-			ptr.Elem().Set(reflect.MakeMap(et))
-			return ptr
-		}
-	} else {
-		ret.newElem = func() reflect.Value { return reflect.New(et) }
-	}
-	return ret
 }
 
 // multiArgTypeStruct == []S
-func multiArgTypeStruct(et reflect.Type) multiArgType {
+func multiArgTypeStruct(et reflect.Type) *multiArgType {
 	cdc := getCodec(et)
 	toPLS := func(slot reflect.Value) *structPLS {
 		return &structPLS{slot, cdc}
 	}
-	return multiArgType{
+	return &multiArgType{
 		getKey: func(aid, ns string, slot reflect.Value) (*Key, error) {
-			return newKeyObjErr(aid, ns, toPLS(slot))
+			return newKeyObjErr(aid, ns, getMGS(slot.Addr().Interface()))
 		},
 		getPM: func(slot reflect.Value) (PropertyMap, error) {
 			return toPLS(slot).Save(true)
 		},
 		getMetaPM: func(slot reflect.Value) PropertyMap {
-			if slot.Type().Implements(typeOfMGS) {
-				return slot.Interface().(MetaGetterSetter).GetAllMeta()
-			}
-			return toPLS(slot).GetAllMeta()
+			return getMGS(slot.Addr().Interface()).GetAllMeta()
 		},
 		setPM: func(slot reflect.Value, pm PropertyMap) error {
 			return toPLS(slot).Load(pm)
@@ -183,23 +176,20 @@ func multiArgTypeStruct(et reflect.Type) multiArgType {
 }
 
 // multiArgTypeStructPtr == []*S
-func multiArgTypeStructPtr(et reflect.Type) multiArgType {
-	cdc := getCodec(et)
+func multiArgTypeStructPtr(et reflect.Type) *multiArgType {
+	cdc := getCodec(et.Elem())
 	toPLS := func(slot reflect.Value) *structPLS {
 		return &structPLS{slot.Elem(), cdc}
 	}
-	return multiArgType{
+	return &multiArgType{
 		getKey: func(aid, ns string, slot reflect.Value) (*Key, error) {
-			return newKeyObjErr(aid, ns, toPLS(slot))
+			return newKeyObjErr(aid, ns, getMGS(slot.Interface()))
 		},
 		getPM: func(slot reflect.Value) (PropertyMap, error) {
 			return toPLS(slot).Save(true)
 		},
 		getMetaPM: func(slot reflect.Value) PropertyMap {
-			if slot.Elem().Type().Implements(typeOfMGS) {
-				return getMGS(slot.Interface()).GetAllMeta()
-			}
-			return toPLS(slot).GetAllMeta()
+			return getMGS(slot.Interface()).GetAllMeta()
 		},
 		setPM: func(slot reflect.Value, pm PropertyMap) error {
 			return toPLS(slot).Load(pm)
@@ -208,16 +198,16 @@ func multiArgTypeStructPtr(et reflect.Type) multiArgType {
 			PopulateKey(toPLS(slot), k)
 		},
 		newElem: func() reflect.Value {
-			return reflect.New(et)
+			return reflect.New(et.Elem())
 		},
 	}
 }
 
 // multiArgTypeInterface == []I
-func multiArgTypeInterface() multiArgType {
-	return multiArgType{
+func multiArgTypeInterface() *multiArgType {
+	return &multiArgType{
 		getKey: func(aid, ns string, slot reflect.Value) (*Key, error) {
-			return newKeyObjErr(aid, ns, slot.Elem().Interface())
+			return newKeyObjErr(aid, ns, getMGS(slot.Elem().Interface()))
 		},
 		getPM: func(slot reflect.Value) (PropertyMap, error) {
 			return mkPLS(slot.Elem().Interface()).Save(true)
@@ -234,24 +224,34 @@ func multiArgTypeInterface() multiArgType {
 	}
 }
 
-func newKeyObjErr(aid, ns string, src interface{}) (*Key, error) {
-	pls := getMGS(src)
-	if key, _ := GetMetaDefault(pls, "key", nil).(*Key); key != nil {
+// multiArgTypeKeyExtraction == *Key
+//
+// This ONLY implements getKey.
+func multiArgTypeKeyExtraction() *multiArgType {
+	return &multiArgType{
+		getKey: func(aid, ns string, slot reflect.Value) (*Key, error) {
+			return slot.Interface().(*Key), nil
+		},
+	}
+}
+
+func newKeyObjErr(aid, ns string, mgs MetaGetterSetter) (*Key, error) {
+	if key, _ := GetMetaDefault(mgs, "key", nil).(*Key); key != nil {
 		return key, nil
 	}
 
 	// get kind
-	kind := GetMetaDefault(pls, "kind", "").(string)
+	kind := GetMetaDefault(mgs, "kind", "").(string)
 	if kind == "" {
-		return nil, fmt.Errorf("unable to extract $kind from %T", src)
+		return nil, errors.New("unable to extract $kind")
 	}
 
 	// get id - allow both to be default for default keys
-	sid := GetMetaDefault(pls, "id", "").(string)
-	iid := GetMetaDefault(pls, "id", 0).(int64)
+	sid := GetMetaDefault(mgs, "id", "").(string)
+	iid := GetMetaDefault(mgs, "id", 0).(int64)
 
 	// get parent
-	par, _ := GetMetaDefault(pls, "parent", nil).(*Key)
+	par, _ := GetMetaDefault(mgs, "parent", nil).(*Key)
 
 	return NewKey(aid, ns, kind, sid, iid, par), nil
 }
@@ -262,3 +262,275 @@ func mkPLS(o interface{}) PropertyLoadSaver {
 	}
 	return GetPLS(o)
 }
+
+func isOKSingleType(t reflect.Type, allowKeys bool) error {
+	switch {
+	case t == nil:
+		return errors.New("no type information")
+	case t.Implements(typeOfPropertyLoadSaver):
+		return nil
+	case !allowKeys && t == typeOfKey:
+		return errors.New("not user datatype")
+	case t.Kind() != reflect.Ptr:
+		return errors.New("not a pointer")
+	case t.Elem().Kind() != reflect.Struct:
+		return errors.New("does not point to a struct")
+	default:
+		return nil
+	}
+}
+
+type metaMultiArgElement struct {
+	arg  reflect.Value
+	mat  *multiArgType
+	size int // size is -1 if this element is not a slice.
+}
+
+type metaMultiArg struct {
+	elems    []metaMultiArgElement
+	keysOnly bool
+
+	count int // total number of elements, flattening slices
+}
+
+func makeMetaMultiArg(args []interface{}, keysOnly bool) (*metaMultiArg, error) {
+	mma := metaMultiArg{
+		elems:    make([]metaMultiArgElement, len(args)),
+		keysOnly: keysOnly,
+	}
+
+	lme := errors.NewLazyMultiError(len(args))
+	for i, arg := range args {
+		if arg == nil {
+			lme.Assign(i, errors.New("cannot use nil as single argument"))
+			continue
+		}
+
+		v := reflect.ValueOf(arg)
+		vt := v.Type()
+		mma.elems[i].arg = v
+
+		// Try and treat the argument as a single-value first. This allows slices
+		// that implement PropertyLoadSaver to be properly treated as a single
+		// element.
+		var err error
+		isSlice := false
+		mat := parseArg(vt, keysOnly)
+		if mat == nil {
+			// If this is a slice, treat it as a slice of arg candidates.
+			if v.Kind() == reflect.Slice {
+				isSlice = true
+				mat = parseArg(vt.Elem(), keysOnly)
+			}
+		} else {
+			// Single types need to be able to be assigned to.
+			err = isOKSingleType(vt, keysOnly)
+		}
+		if mat == nil {
+			err = errors.New("not a PLS or pointer-to-struct")
+		}
+		if err != nil {
+			lme.Assign(i, fmt.Errorf("invalid input type (%T): %s", arg, err))
+			continue
+		}
+
+		mma.elems[i].mat = mat
+		if isSlice {
+			l := v.Len()
+			mma.count += l
+			mma.elems[i].size = l
+		} else {
+			mma.count++
+			mma.elems[i].size = -1
+		}
+	}
+	if err := lme.Get(); err != nil {
+		return nil, err
+	}
+
+	return &mma, nil
+}
+
+func (mma *metaMultiArg) iterator(cb metaMultiArgIteratorCallback) *metaMultiArgIterator {
+	return &metaMultiArgIterator{
+		metaMultiArg: mma,
+		cb:           cb,
+	}
+}
+
+// getKeysPMs returns the
+func (mma *metaMultiArg) getKeysPMs(aid, ns string, meta bool) ([]*Key, []PropertyMap, error) {
+	var et errorTracker
+	it := mma.iterator(et.init(mma))
+
+	// Determine our flattened keys and property maps.
+	retKey := make([]*Key, mma.count)
+	var retPM []PropertyMap
+	if !mma.keysOnly {
+		retPM = make([]PropertyMap, mma.count)
+	}
+
+	for i := 0; i < mma.count; i++ {
+		it.next(func(mat *multiArgType, slot reflect.Value) error {
+			key, err := mat.getKey(aid, ns, slot)
+			if err != nil {
+				return err
+			}
+			retKey[i] = key
+
+			if !mma.keysOnly {
+				var pm PropertyMap
+				if meta {
+					pm = mat.getMetaPM(slot)
+				} else {
+					var err error
+					if pm, err = mat.getPM(slot); err != nil {
+						return err
+					}
+				}
+				retPM[i] = pm
+			}
+			return nil
+		})
+	}
+	return retKey, retPM, et.error()
+}
+
+type metaMultiArgIterator struct {
+	*metaMultiArg
+
+	cb metaMultiArgIteratorCallback
+
+	index   int // flattened index
+	elemIdx int // current index in slice
+	slotIdx int // current index within elemIdx element (0 if single)
+}
+
+func (mac *metaMultiArgIterator) next(fn func(*multiArgType, reflect.Value) error) {

[iannucci, 2016/05/28 02:50:53]

s/mac/it ?

[dnj, 2016/05/28 17:47:21]

Done.

+	if mac.remaining() <= 0 {
+		panic("out of bounds")
+	}
+
+	// Advance to the next populated element/slot.
+	elem := &mac.elems[mac.elemIdx]
+	if mac.index > 0 {
+		for {
+			mac.slotIdx++
+			if mac.slotIdx >= elem.size {
+				mac.elemIdx++
+				mac.slotIdx = 0
+				elem = &mac.elems[mac.elemIdx]
+			}
+
+			// We're done iterating, unless we're on a zero-sized slice element.
+			if elem.size != 0 {
+				break
+			}
+		}
+	}
+
+	// Get the current slot value.
+	slot := elem.arg
+	if elem.size >= 0 {
+		// slot is a slice type, get its member.
+		slot = slot.Index(mac.slotIdx)
+	}
+
+	// Execute our callback.
+	mac.cb(mac, fn(elem.mat, slot))
+
+	// Advance our flattened index.
+	mac.index++
+}
+
+func (mac *metaMultiArgIterator) remaining() int {
+	return mac.count - mac.index
+}
+
+type metaMultiArgIteratorCallback func(*metaMultiArgIterator, error)
+
+type errorTracker struct {
+	elemErrors  errors.LazyMultiError
+	sliceErrors map[int]errors.LazyMultiError
+}
+
+func (et *errorTracker) init(mma *metaMultiArg) metaMultiArgIteratorCallback {
+	et.elemErrors = errors.NewLazyMultiError(len(mma.elems))
+	return et.trackError
+}
+
+func (et *errorTracker) trackError(it *metaMultiArgIterator, err error) {
+	if err == nil {
+		return
+	}
+
+	// If this is a single element, assign the error directly.
+	elem := it.elems[it.elemIdx]
+	if elem.size < 0 {
+		et.elemErrors.Assign(it.elemIdx, err)
+	} else {
+		// This is a slice element. Use a slice-sized MultiError for its element
+		// error slot, then add this error to the inner MultiError's slot index.
+		ilme := et.sliceErrors[it.elemIdx]
+		if ilme == nil {
+			ilme = errors.NewLazyMultiError(elem.size)
+
+			if et.sliceErrors == nil {
+				et.sliceErrors = make(map[int]errors.LazyMultiError)

[iannucci, 2016/05/28 02:50:53]

let's just allocate the full 1st dimension map on the first non-nil error 

[dnj, 2016/05/28 17:47:21]

Am building in-place. There is a bit of reflection per error, but I think this
is probably not worse than a hash lookup.

+			}
+			et.sliceErrors[it.elemIdx] = ilme
+		}
+		ilme.Assign(it.slotIdx, err)
+	}
+}
+
+func (et *errorTracker) error() error {
+	for i, lme := range et.sliceErrors {
+		if lerr := lme.Get(); lerr != nil {
+			et.elemErrors.Assign(i, lerr)
+		}
+	}
+	et.sliceErrors = nil
+	return et.elemErrors.Get()
+}
+
+type boolTracker struct {
+	errorTracker
+
+	res ExistsResult
+}
+
+func (bt *boolTracker) init(mma *metaMultiArg) metaMultiArgIteratorCallback {
+	bt.errorTracker.init(mma)
+
+	sizes := make([]int, len(mma.elems))
+	for i, e := range mma.elems {
+		if e.size < 0 {
+			sizes[i] = 1
+		} else {
+			sizes[i] = e.size
+		}
+	}
+
+	bt.res.init(sizes...)
+	return bt.trackExistsResult
+}
+
+func (bt *boolTracker) trackExistsResult(it *metaMultiArgIterator, err error) {
+	switch err {
+	case nil:
+		bt.res.set(it.elemIdx, it.slotIdx)
+
+	case ErrNoSuchEntity:
+		break
+
+	default:
+		// Pass through to track as MultiError.
+		bt.errorTracker.trackError(it, err)
+	}
+}
+
+func (bt *boolTracker) result() *ExistsResult {
+	bt.res.updateSlices()
+	return &bt.res
+}