mirror of https://github.com/go-gitea/gitea
add other session providers (#5963)
techknowlogick
6 years ago
committed by
GitHub
parent
bf4badad1d
commit
9de871a0f8
@ -0,0 +1,21 @@ |
||||
The MIT License (MIT) |
||||
|
||||
Copyright (c) 2013 TOML authors |
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||
of this software and associated documentation files (the "Software"), to deal |
||||
in the Software without restriction, including without limitation the rights |
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||
copies of the Software, and to permit persons to whom the Software is |
||||
furnished to do so, subject to the following conditions: |
||||
|
||||
The above copyright notice and this permission notice shall be included in |
||||
all copies or substantial portions of the Software. |
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
||||
THE SOFTWARE. |
||||
@ -0,0 +1,21 @@ |
||||
The MIT License (MIT) |
||||
|
||||
Copyright (c) 2013 TOML authors |
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||
of this software and associated documentation files (the "Software"), to deal |
||||
in the Software without restriction, including without limitation the rights |
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||
copies of the Software, and to permit persons to whom the Software is |
||||
furnished to do so, subject to the following conditions: |
||||
|
||||
The above copyright notice and this permission notice shall be included in |
||||
all copies or substantial portions of the Software. |
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
||||
THE SOFTWARE. |
||||
@ -0,0 +1,21 @@ |
||||
The MIT License (MIT) |
||||
|
||||
Copyright (c) 2013 TOML authors |
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||
of this software and associated documentation files (the "Software"), to deal |
||||
in the Software without restriction, including without limitation the rights |
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||
copies of the Software, and to permit persons to whom the Software is |
||||
furnished to do so, subject to the following conditions: |
||||
|
||||
The above copyright notice and this permission notice shall be included in |
||||
all copies or substantial portions of the Software. |
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
||||
THE SOFTWARE. |
||||
@ -0,0 +1,21 @@ |
||||
The MIT License (MIT) |
||||
|
||||
Copyright (c) 2013 TOML authors |
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||
of this software and associated documentation files (the "Software"), to deal |
||||
in the Software without restriction, including without limitation the rights |
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||
copies of the Software, and to permit persons to whom the Software is |
||||
furnished to do so, subject to the following conditions: |
||||
|
||||
The above copyright notice and this permission notice shall be included in |
||||
all copies or substantial portions of the Software. |
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
||||
THE SOFTWARE. |
||||
@ -0,0 +1,509 @@ |
||||
package toml |
||||
|
||||
import ( |
||||
"fmt" |
||||
"io" |
||||
"io/ioutil" |
||||
"math" |
||||
"reflect" |
||||
"strings" |
||||
"time" |
||||
) |
||||
|
||||
func e(format string, args ...interface{}) error { |
||||
return fmt.Errorf("toml: "+format, args...) |
||||
} |
||||
|
||||
// Unmarshaler is the interface implemented by objects that can unmarshal a
|
||||
// TOML description of themselves.
|
||||
type Unmarshaler interface { |
||||
UnmarshalTOML(interface{}) error |
||||
} |
||||
|
||||
// Unmarshal decodes the contents of `p` in TOML format into a pointer `v`.
|
||||
func Unmarshal(p []byte, v interface{}) error { |
||||
_, err := Decode(string(p), v) |
||||
return err |
||||
} |
||||
|
||||
// Primitive is a TOML value that hasn't been decoded into a Go value.
|
||||
// When using the various `Decode*` functions, the type `Primitive` may
|
||||
// be given to any value, and its decoding will be delayed.
|
||||
//
|
||||
// A `Primitive` value can be decoded using the `PrimitiveDecode` function.
|
||||
//
|
||||
// The underlying representation of a `Primitive` value is subject to change.
|
||||
// Do not rely on it.
|
||||
//
|
||||
// N.B. Primitive values are still parsed, so using them will only avoid
|
||||
// the overhead of reflection. They can be useful when you don't know the
|
||||
// exact type of TOML data until run time.
|
||||
type Primitive struct { |
||||
undecoded interface{} |
||||
context Key |
||||
} |
||||
|
||||
// DEPRECATED!
|
||||
//
|
||||
// Use MetaData.PrimitiveDecode instead.
|
||||
func PrimitiveDecode(primValue Primitive, v interface{}) error { |
||||
md := MetaData{decoded: make(map[string]bool)} |
||||
return md.unify(primValue.undecoded, rvalue(v)) |
||||
} |
||||
|
||||
// PrimitiveDecode is just like the other `Decode*` functions, except it
|
||||
// decodes a TOML value that has already been parsed. Valid primitive values
|
||||
// can *only* be obtained from values filled by the decoder functions,
|
||||
// including this method. (i.e., `v` may contain more `Primitive`
|
||||
// values.)
|
||||
//
|
||||
// Meta data for primitive values is included in the meta data returned by
|
||||
// the `Decode*` functions with one exception: keys returned by the Undecoded
|
||||
// method will only reflect keys that were decoded. Namely, any keys hidden
|
||||
// behind a Primitive will be considered undecoded. Executing this method will
|
||||
// update the undecoded keys in the meta data. (See the example.)
|
||||
func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error { |
||||
md.context = primValue.context |
||||
defer func() { md.context = nil }() |
||||
return md.unify(primValue.undecoded, rvalue(v)) |
||||
} |
||||
|
||||
// Decode will decode the contents of `data` in TOML format into a pointer
|
||||
// `v`.
|
||||
//
|
||||
// TOML hashes correspond to Go structs or maps. (Dealer's choice. They can be
|
||||
// used interchangeably.)
|
||||
//
|
||||
// TOML arrays of tables correspond to either a slice of structs or a slice
|
||||
// of maps.
|
||||
//
|
||||
// TOML datetimes correspond to Go `time.Time` values.
|
||||
//
|
||||
// All other TOML types (float, string, int, bool and array) correspond
|
||||
// to the obvious Go types.
|
||||
//
|
||||
// An exception to the above rules is if a type implements the
|
||||
// encoding.TextUnmarshaler interface. In this case, any primitive TOML value
|
||||
// (floats, strings, integers, booleans and datetimes) will be converted to
|
||||
// a byte string and given to the value's UnmarshalText method. See the
|
||||
// Unmarshaler example for a demonstration with time duration strings.
|
||||
//
|
||||
// Key mapping
|
||||
//
|
||||
// TOML keys can map to either keys in a Go map or field names in a Go
|
||||
// struct. The special `toml` struct tag may be used to map TOML keys to
|
||||
// struct fields that don't match the key name exactly. (See the example.)
|
||||
// A case insensitive match to struct names will be tried if an exact match
|
||||
// can't be found.
|
||||
//
|
||||
// The mapping between TOML values and Go values is loose. That is, there
|
||||
// may exist TOML values that cannot be placed into your representation, and
|
||||
// there may be parts of your representation that do not correspond to
|
||||
// TOML values. This loose mapping can be made stricter by using the IsDefined
|
||||
// and/or Undecoded methods on the MetaData returned.
|
||||
//
|
||||
// This decoder will not handle cyclic types. If a cyclic type is passed,
|
||||
// `Decode` will not terminate.
|
||||
func Decode(data string, v interface{}) (MetaData, error) { |
||||
rv := reflect.ValueOf(v) |
||||
if rv.Kind() != reflect.Ptr { |
||||
return MetaData{}, e("Decode of non-pointer %s", reflect.TypeOf(v)) |
||||
} |
||||
if rv.IsNil() { |
||||
return MetaData{}, e("Decode of nil %s", reflect.TypeOf(v)) |
||||
} |
||||
p, err := parse(data) |
||||
if err != nil { |
||||
return MetaData{}, err |
||||
} |
||||
md := MetaData{ |
||||
p.mapping, p.types, p.ordered, |
||||
make(map[string]bool, len(p.ordered)), nil, |
||||
} |
||||
return md, md.unify(p.mapping, indirect(rv)) |
||||
} |
||||
|
||||
// DecodeFile is just like Decode, except it will automatically read the
|
||||
// contents of the file at `fpath` and decode it for you.
|
||||
func DecodeFile(fpath string, v interface{}) (MetaData, error) { |
||||
bs, err := ioutil.ReadFile(fpath) |
||||
if err != nil { |
||||
return MetaData{}, err |
||||
} |
||||
return Decode(string(bs), v) |
||||
} |
||||
|
||||
// DecodeReader is just like Decode, except it will consume all bytes
|
||||
// from the reader and decode it for you.
|
||||
func DecodeReader(r io.Reader, v interface{}) (MetaData, error) { |
||||
bs, err := ioutil.ReadAll(r) |
||||
if err != nil { |
||||
return MetaData{}, err |
||||
} |
||||
return Decode(string(bs), v) |
||||
} |
||||
|
||||
// unify performs a sort of type unification based on the structure of `rv`,
|
||||
// which is the client representation.
|
||||
//
|
||||
// Any type mismatch produces an error. Finding a type that we don't know
|
||||
// how to handle produces an unsupported type error.
|
||||
func (md *MetaData) unify(data interface{}, rv reflect.Value) error { |
||||
|
||||
// Special case. Look for a `Primitive` value.
|
||||
if rv.Type() == reflect.TypeOf((*Primitive)(nil)).Elem() { |
||||
// Save the undecoded data and the key context into the primitive
|
||||
// value.
|
||||
context := make(Key, len(md.context)) |
||||
copy(context, md.context) |
||||
rv.Set(reflect.ValueOf(Primitive{ |
||||
undecoded: data, |
||||
context: context, |
||||
})) |
||||
return nil |
||||
} |
||||
|
||||
// Special case. Unmarshaler Interface support.
|
||||
if rv.CanAddr() { |
||||
if v, ok := rv.Addr().Interface().(Unmarshaler); ok { |
||||
return v.UnmarshalTOML(data) |
||||
} |
||||
} |
||||
|
||||
// Special case. Handle time.Time values specifically.
|
||||
// TODO: Remove this code when we decide to drop support for Go 1.1.
|
||||
// This isn't necessary in Go 1.2 because time.Time satisfies the encoding
|
||||
// interfaces.
|
||||
if rv.Type().AssignableTo(rvalue(time.Time{}).Type()) { |
||||
return md.unifyDatetime(data, rv) |
||||
} |
||||
|
||||
// Special case. Look for a value satisfying the TextUnmarshaler interface.
|
||||
if v, ok := rv.Interface().(TextUnmarshaler); ok { |
||||
return md.unifyText(data, v) |
||||
} |
||||
// BUG(burntsushi)
|
||||
// The behavior here is incorrect whenever a Go type satisfies the
|
||||
// encoding.TextUnmarshaler interface but also corresponds to a TOML
|
||||
// hash or array. In particular, the unmarshaler should only be applied
|
||||
// to primitive TOML values. But at this point, it will be applied to
|
||||
// all kinds of values and produce an incorrect error whenever those values
|
||||
// are hashes or arrays (including arrays of tables).
|
||||
|
||||
k := rv.Kind() |
||||
|
||||
// laziness
|
||||
if k >= reflect.Int && k <= reflect.Uint64 { |
||||
return md.unifyInt(data, rv) |
||||
} |
||||
switch k { |
||||
case reflect.Ptr: |
||||
elem := reflect.New(rv.Type().Elem()) |
||||
err := md.unify(data, reflect.Indirect(elem)) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
rv.Set(elem) |
||||
return nil |
||||
case reflect.Struct: |
||||
return md.unifyStruct(data, rv) |
||||
case reflect.Map: |
||||
return md.unifyMap(data, rv) |
||||
case reflect.Array: |
||||
return md.unifyArray(data, rv) |
||||
case reflect.Slice: |
||||
return md.unifySlice(data, rv) |
||||
case reflect.String: |
||||
return md.unifyString(data, rv) |
||||
case reflect.Bool: |
||||
return md.unifyBool(data, rv) |
||||
case reflect.Interface: |
||||
// we only support empty interfaces.
|
||||
if rv.NumMethod() > 0 { |
||||
return e("unsupported type %s", rv.Type()) |
||||
} |
||||
return md.unifyAnything(data, rv) |
||||
case reflect.Float32: |
||||
fallthrough |
||||
case reflect.Float64: |
||||
return md.unifyFloat64(data, rv) |
||||
} |
||||
return e("unsupported type %s", rv.Kind()) |
||||
} |
||||
|
||||
func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { |
||||
tmap, ok := mapping.(map[string]interface{}) |
||||
if !ok { |
||||
if mapping == nil { |
||||
return nil |
||||
} |
||||
return e("type mismatch for %s: expected table but found %T", |
||||
rv.Type().String(), mapping) |
||||
} |
||||
|
||||
for key, datum := range tmap { |
||||
var f *field |
||||
fields := cachedTypeFields(rv.Type()) |
||||
for i := range fields { |
||||
ff := &fields[i] |
||||
if ff.name == key { |
||||
f = ff |
||||
break |
||||
} |
||||
if f == nil && strings.EqualFold(ff.name, key) { |
||||
f = ff |
||||
} |
||||
} |
||||
if f != nil { |
||||
subv := rv |
||||
for _, i := range f.index { |
||||
subv = indirect(subv.Field(i)) |
||||
} |
||||
if isUnifiable(subv) { |
||||
md.decoded[md.context.add(key).String()] = true |
||||
md.context = append(md.context, key) |
||||
if err := md.unify(datum, subv); err != nil { |
||||
return err |
||||
} |
||||
md.context = md.context[0 : len(md.context)-1] |
||||
} else if f.name != "" { |
||||
// Bad user! No soup for you!
|
||||
return e("cannot write unexported field %s.%s", |
||||
rv.Type().String(), f.name) |
||||
} |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (md *MetaData) unifyMap(mapping interface{}, rv reflect.Value) error { |
||||
tmap, ok := mapping.(map[string]interface{}) |
||||
if !ok { |
||||
if tmap == nil { |
||||
return nil |
||||
} |
||||
return badtype("map", mapping) |
||||
} |
||||
if rv.IsNil() { |
||||
rv.Set(reflect.MakeMap(rv.Type())) |
||||
} |
||||
for k, v := range tmap { |
||||
md.decoded[md.context.add(k).String()] = true |
||||
md.context = append(md.context, k) |
||||
|
||||
rvkey := indirect(reflect.New(rv.Type().Key())) |
||||
rvval := reflect.Indirect(reflect.New(rv.Type().Elem())) |
||||
if err := md.unify(v, rvval); err != nil { |
||||
return err |
||||
} |
||||
md.context = md.context[0 : len(md.context)-1] |
||||
|
||||
rvkey.SetString(k) |
||||
rv.SetMapIndex(rvkey, rvval) |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (md *MetaData) unifyArray(data interface{}, rv reflect.Value) error { |
||||
datav := reflect.ValueOf(data) |
||||
if datav.Kind() != reflect.Slice { |
||||
if !datav.IsValid() { |
||||
return nil |
||||
} |
||||
return badtype("slice", data) |
||||
} |
||||
sliceLen := datav.Len() |
||||
if sliceLen != rv.Len() { |
||||
return e("expected array length %d; got TOML array of length %d", |
||||
rv.Len(), sliceLen) |
||||
} |
||||
return md.unifySliceArray(datav, rv) |
||||
} |
||||
|
||||
func (md *MetaData) unifySlice(data interface{}, rv reflect.Value) error { |
||||
datav := reflect.ValueOf(data) |
||||
if datav.Kind() != reflect.Slice { |
||||
if !datav.IsValid() { |
||||
return nil |
||||
} |
||||
return badtype("slice", data) |
||||
} |
||||
n := datav.Len() |
||||
if rv.IsNil() || rv.Cap() < n { |
||||
rv.Set(reflect.MakeSlice(rv.Type(), n, n)) |
||||
} |
||||
rv.SetLen(n) |
||||
return md.unifySliceArray(datav, rv) |
||||
} |
||||
|
||||
func (md *MetaData) unifySliceArray(data, rv reflect.Value) error { |
||||
sliceLen := data.Len() |
||||
for i := 0; i < sliceLen; i++ { |
||||
v := data.Index(i).Interface() |
||||
sliceval := indirect(rv.Index(i)) |
||||
if err := md.unify(v, sliceval); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (md *MetaData) unifyDatetime(data interface{}, rv reflect.Value) error { |
||||
if _, ok := data.(time.Time); ok { |
||||
rv.Set(reflect.ValueOf(data)) |
||||
return nil |
||||
} |
||||
return badtype("time.Time", data) |
||||
} |
||||
|
||||
func (md *MetaData) unifyString(data interface{}, rv reflect.Value) error { |
||||
if s, ok := data.(string); ok { |
||||
rv.SetString(s) |
||||
return nil |
||||
} |
||||
return badtype("string", data) |
||||
} |
||||
|
||||
func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error { |
||||
if num, ok := data.(float64); ok { |
||||
switch rv.Kind() { |
||||
case reflect.Float32: |
||||
fallthrough |
||||
case reflect.Float64: |
||||
rv.SetFloat(num) |
||||
default: |
||||
panic("bug") |
||||
} |
||||
return nil |
||||
} |
||||
return badtype("float", data) |
||||
} |
||||
|
||||
func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { |
||||
if num, ok := data.(int64); ok { |
||||
if rv.Kind() >= reflect.Int && rv.Kind() <= reflect.Int64 { |
||||
switch rv.Kind() { |
||||
case reflect.Int, reflect.Int64: |
||||
// No bounds checking necessary.
|
||||
case reflect.Int8: |
||||
if num < math.MinInt8 || num > math.MaxInt8 { |
||||
return e("value %d is out of range for int8", num) |
||||
} |
||||
case reflect.Int16: |
||||
if num < math.MinInt16 || num > math.MaxInt16 { |
||||
return e("value %d is out of range for int16", num) |
||||
} |
||||
case reflect.Int32: |
||||
if num < math.MinInt32 || num > math.MaxInt32 { |
||||
return e("value %d is out of range for int32", num) |
||||
} |
||||
} |
||||
rv.SetInt(num) |
||||
} else if rv.Kind() >= reflect.Uint && rv.Kind() <= reflect.Uint64 { |
||||
unum := uint64(num) |
||||
switch rv.Kind() { |
||||
case reflect.Uint, reflect.Uint64: |
||||
// No bounds checking necessary.
|
||||
case reflect.Uint8: |
||||
if num < 0 || unum > math.MaxUint8 { |
||||
return e("value %d is out of range for uint8", num) |
||||
} |
||||
case reflect.Uint16: |
||||
if num < 0 || unum > math.MaxUint16 { |
||||
return e("value %d is out of range for uint16", num) |
||||
} |
||||
case reflect.Uint32: |
||||
if num < 0 || unum > math.MaxUint32 { |
||||
return e("value %d is out of range for uint32", num) |
||||
} |
||||
} |
||||
rv.SetUint(unum) |
||||
} else { |
||||
panic("unreachable") |
||||
} |
||||
return nil |
||||
} |
||||
return badtype("integer", data) |
||||
} |
||||
|
||||
func (md *MetaData) unifyBool(data interface{}, rv reflect.Value) error { |
||||
if b, ok := data.(bool); ok { |
||||
rv.SetBool(b) |
||||
return nil |
||||
} |
||||
return badtype("boolean", data) |
||||
} |
||||
|
||||
func (md *MetaData) unifyAnything(data interface{}, rv reflect.Value) error { |
||||
rv.Set(reflect.ValueOf(data)) |
||||
return nil |
||||
} |
||||
|
||||
func (md *MetaData) unifyText(data interface{}, v TextUnmarshaler) error { |
||||
var s string |
||||
switch sdata := data.(type) { |
||||
case TextMarshaler: |
||||
text, err := sdata.MarshalText() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
s = string(text) |
||||
case fmt.Stringer: |
||||
s = sdata.String() |
||||
case string: |
||||
s = sdata |
||||
case bool: |
||||
s = fmt.Sprintf("%v", sdata) |
||||
case int64: |
||||
s = fmt.Sprintf("%d", sdata) |
||||
case float64: |
||||
s = fmt.Sprintf("%f", sdata) |
||||
default: |
||||
return badtype("primitive (string-like)", data) |
||||
} |
||||
if err := v.UnmarshalText([]byte(s)); err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// rvalue returns a reflect.Value of `v`. All pointers are resolved.
|
||||
func rvalue(v interface{}) reflect.Value { |
||||
return indirect(reflect.ValueOf(v)) |
||||
} |
||||
|
||||
// indirect returns the value pointed to by a pointer.
|
||||
// Pointers are followed until the value is not a pointer.
|
||||
// New values are allocated for each nil pointer.
|
||||
//
|
||||
// An exception to this rule is if the value satisfies an interface of
|
||||
// interest to us (like encoding.TextUnmarshaler).
|
||||
func indirect(v reflect.Value) reflect.Value { |
||||
if v.Kind() != reflect.Ptr { |
||||
if v.CanSet() { |
||||
pv := v.Addr() |
||||
if _, ok := pv.Interface().(TextUnmarshaler); ok { |
||||
return pv |
||||
} |
||||
} |
||||
return v |
||||
} |
||||
if v.IsNil() { |
||||
v.Set(reflect.New(v.Type().Elem())) |
||||
} |
||||
return indirect(reflect.Indirect(v)) |
||||
} |
||||
|
||||
func isUnifiable(rv reflect.Value) bool { |
||||
if rv.CanSet() { |
||||
return true |
||||
} |
||||
if _, ok := rv.Interface().(TextUnmarshaler); ok { |
||||
return true |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func badtype(expected string, data interface{}) error { |
||||
return e("cannot load TOML value of type %T into a Go %s", data, expected) |
||||
} |
||||
@ -0,0 +1,121 @@ |
||||
package toml |
||||
|
||||
import "strings" |
||||
|
||||
// MetaData allows access to meta information about TOML data that may not
|
||||
// be inferrable via reflection. In particular, whether a key has been defined
|
||||
// and the TOML type of a key.
|
||||
type MetaData struct { |
||||
mapping map[string]interface{} |
||||
types map[string]tomlType |
||||
keys []Key |
||||
decoded map[string]bool |
||||
context Key // Used only during decoding.
|
||||
} |
||||
|
||||
// IsDefined returns true if the key given exists in the TOML data. The key
|
||||
// should be specified hierarchially. e.g.,
|
||||
//
|
||||
// // access the TOML key 'a.b.c'
|
||||
// IsDefined("a", "b", "c")
|
||||
//
|
||||
// IsDefined will return false if an empty key given. Keys are case sensitive.
|
||||
func (md *MetaData) IsDefined(key ...string) bool { |
||||
if len(key) == 0 { |
||||
return false |
||||
} |
||||
|
||||
var hash map[string]interface{} |
||||
var ok bool |
||||
var hashOrVal interface{} = md.mapping |
||||
for _, k := range key { |
||||
if hash, ok = hashOrVal.(map[string]interface{}); !ok { |
||||
return false |
||||
} |
||||
if hashOrVal, ok = hash[k]; !ok { |
||||
return false |
||||
} |
||||
} |
||||
return true |
||||
} |
||||
|
||||
// Type returns a string representation of the type of the key specified.
|
||||
//
|
||||
// Type will return the empty string if given an empty key or a key that
|
||||
// does not exist. Keys are case sensitive.
|
||||
func (md *MetaData) Type(key ...string) string { |
||||
fullkey := strings.Join(key, ".") |
||||
if typ, ok := md.types[fullkey]; ok { |
||||
return typ.typeString() |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
// Key is the type of any TOML key, including key groups. Use (MetaData).Keys
|
||||
// to get values of this type.
|
||||
type Key []string |
||||
|
||||
func (k Key) String() string { |
||||
return strings.Join(k, ".") |
||||
} |
||||
|
||||
func (k Key) maybeQuotedAll() string { |
||||
var ss []string |
||||
for i := range k { |
||||
ss = append(ss, k.maybeQuoted(i)) |
||||
} |
||||
return strings.Join(ss, ".") |
||||
} |
||||
|
||||
func (k Key) maybeQuoted(i int) string { |
||||
quote := false |
||||
for _, c := range k[i] { |
||||
if !isBareKeyChar(c) { |
||||
quote = true |
||||
break |
||||
} |
||||
} |
||||
if quote { |
||||
return "\"" + strings.Replace(k[i], "\"", "\\\"", -1) + "\"" |
||||
} |
||||
return k[i] |
||||
} |
||||
|
||||
func (k Key) add(piece string) Key { |
||||
newKey := make(Key, len(k)+1) |
||||
copy(newKey, k) |
||||
newKey[len(k)] = piece |
||||
return newKey |
||||
} |
||||
|
||||
// Keys returns a slice of every key in the TOML data, including key groups.
|
||||
// Each key is itself a slice, where the first element is the top of the
|
||||
// hierarchy and the last is the most specific.
|
||||
//
|
||||
// The list will have the same order as the keys appeared in the TOML data.
|
||||
//
|
||||
// All keys returned are non-empty.
|
||||
func (md *MetaData) Keys() []Key { |
||||
return md.keys |
||||
} |
||||
|
||||
// Undecoded returns all keys that have not been decoded in the order in which
|
||||
// they appear in the original TOML document.
|
||||
//
|
||||
// This includes keys that haven't been decoded because of a Primitive value.
|
||||
// Once the Primitive value is decoded, the keys will be considered decoded.
|
||||
//
|
||||
// Also note that decoding into an empty interface will result in no decoding,
|
||||
// and so no keys will be considered decoded.
|
||||
//
|
||||
// In this sense, the Undecoded keys correspond to keys in the TOML document
|
||||
// that do not have a concrete type in your representation.
|
||||
func (md *MetaData) Undecoded() []Key { |
||||
undecoded := make([]Key, 0, len(md.keys)) |
||||
for _, key := range md.keys { |
||||
if !md.decoded[key.String()] { |
||||
undecoded = append(undecoded, key) |
||||
} |
||||
} |
||||
return undecoded |
||||
} |
||||
@ -0,0 +1,27 @@ |
||||
/* |
||||
Package toml provides facilities for decoding and encoding TOML configuration |
||||
files via reflection. There is also support for delaying decoding with |
||||
the Primitive type, and querying the set of keys in a TOML document with the |
||||
MetaData type. |
||||
|
||||
The specification implemented: https://github.com/toml-lang/toml
|
||||
|
||||
The sub-command github.com/BurntSushi/toml/cmd/tomlv can be used to verify |
||||
whether a file is a valid TOML document. It can also be used to print the |
||||
type of each key in a TOML document. |
||||
|
||||
Testing |
||||
|
||||
There are two important types of tests used for this package. The first is |
||||
contained inside '*_test.go' files and uses the standard Go unit testing |
||||
framework. These tests are primarily devoted to holistically testing the |
||||
decoder and encoder. |
||||
|
||||
The second type of testing is used to verify the implementation's adherence |
||||
to the TOML specification. These tests have been factored into their own |
||||
project: https://github.com/BurntSushi/toml-test
|
||||
|
||||
The reason the tests are in a separate project is so that they can be used by |
||||
any implementation of TOML. Namely, it is language agnostic. |
||||
*/ |
||||
package toml |
||||
@ -0,0 +1,568 @@ |
||||
package toml |
||||
|
||||
import ( |
||||
"bufio" |
||||
"errors" |
||||
"fmt" |
||||
"io" |
||||
"reflect" |
||||
"sort" |
||||
"strconv" |
||||
"strings" |
||||
"time" |
||||
) |
||||
|
||||
type tomlEncodeError struct{ error } |
||||
|
||||
var ( |
||||
errArrayMixedElementTypes = errors.New( |
||||
"toml: cannot encode array with mixed element types") |
||||
errArrayNilElement = errors.New( |
||||
"toml: cannot encode array with nil element") |
||||
errNonString = errors.New( |
||||
"toml: cannot encode a map with non-string key type") |
||||
errAnonNonStruct = errors.New( |
||||
"toml: cannot encode an anonymous field that is not a struct") |
||||
errArrayNoTable = errors.New( |
||||
"toml: TOML array element cannot contain a table") |
||||
errNoKey = errors.New( |
||||
"toml: top-level values must be Go maps or structs") |
||||
errAnything = errors.New("") // used in testing
|
||||
) |
||||
|
||||
var quotedReplacer = strings.NewReplacer( |
||||
"\t", "\\t", |
||||
"\n", "\\n", |
||||
"\r", "\\r", |
||||
"\"", "\\\"", |
||||
"\\", "\\\\", |
||||
) |
||||
|
||||
// Encoder controls the encoding of Go values to a TOML document to some
|
||||
// io.Writer.
|
||||
//
|
||||
// The indentation level can be controlled with the Indent field.
|
||||
type Encoder struct { |
||||
// A single indentation level. By default it is two spaces.
|
||||
Indent string |
||||
|
||||
// hasWritten is whether we have written any output to w yet.
|
||||
hasWritten bool |
||||
w *bufio.Writer |
||||
} |
||||
|
||||
// NewEncoder returns a TOML encoder that encodes Go values to the io.Writer
|
||||
// given. By default, a single indentation level is 2 spaces.
|
||||
func NewEncoder(w io.Writer) *Encoder { |
||||
return &Encoder{ |
||||
w: bufio.NewWriter(w), |
||||
Indent: " ", |
||||
} |
||||
} |
||||
|
||||
// Encode writes a TOML representation of the Go value to the underlying
|
||||
// io.Writer. If the value given cannot be encoded to a valid TOML document,
|
||||
// then an error is returned.
|
||||
//
|
||||
// The mapping between Go values and TOML values should be precisely the same
|
||||
// as for the Decode* functions. Similarly, the TextMarshaler interface is
|
||||
// supported by encoding the resulting bytes as strings. (If you want to write
|
||||
// arbitrary binary data then you will need to use something like base64 since
|
||||
// TOML does not have any binary types.)
|
||||
//
|
||||
// When encoding TOML hashes (i.e., Go maps or structs), keys without any
|
||||
// sub-hashes are encoded first.
|
||||
//
|
||||
// If a Go map is encoded, then its keys are sorted alphabetically for
|
||||
// deterministic output. More control over this behavior may be provided if
|
||||
// there is demand for it.
|
||||
//
|
||||
// Encoding Go values without a corresponding TOML representation---like map
|
||||
// types with non-string keys---will cause an error to be returned. Similarly
|
||||
// for mixed arrays/slices, arrays/slices with nil elements, embedded
|
||||
// non-struct types and nested slices containing maps or structs.
|
||||
// (e.g., [][]map[string]string is not allowed but []map[string]string is OK
|
||||
// and so is []map[string][]string.)
|
||||
func (enc *Encoder) Encode(v interface{}) error { |
||||
rv := eindirect(reflect.ValueOf(v)) |
||||
if err := enc.safeEncode(Key([]string{}), rv); err != nil { |
||||
return err |
||||
} |
||||
return enc.w.Flush() |
||||
} |
||||
|
||||
func (enc *Encoder) safeEncode(key Key, rv reflect.Value) (err error) { |
||||
defer func() { |
||||
if r := recover(); r != nil { |
||||
if terr, ok := r.(tomlEncodeError); ok { |
||||
err = terr.error |
||||
return |
||||
} |
||||
panic(r) |
||||
} |
||||
}() |
||||
enc.encode(key, rv) |
||||
return nil |
||||
} |
||||
|
||||
func (enc *Encoder) encode(key Key, rv reflect.Value) { |
||||
// Special case. Time needs to be in ISO8601 format.
|
||||
// Special case. If we can marshal the type to text, then we used that.
|
||||
// Basically, this prevents the encoder for handling these types as
|
||||
// generic structs (or whatever the underlying type of a TextMarshaler is).
|
||||
switch rv.Interface().(type) { |
||||
case time.Time, TextMarshaler: |
||||
enc.keyEqElement(key, rv) |
||||
return |
||||
} |
||||
|
||||
k := rv.Kind() |
||||
switch k { |
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, |
||||
reflect.Int64, |
||||
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, |
||||
reflect.Uint64, |
||||
reflect.Float32, reflect.Float64, reflect.String, reflect.Bool: |
||||
enc.keyEqElement(key, rv) |
||||
case reflect.Array, reflect.Slice: |
||||
if typeEqual(tomlArrayHash, tomlTypeOfGo(rv)) { |
||||
enc.eArrayOfTables(key, rv) |
||||
} else { |
||||
enc.keyEqElement(key, rv) |
||||
} |
||||
case reflect.Interface: |
||||
if rv.IsNil() { |
||||
return |
||||
} |
||||
enc.encode(key, rv.Elem()) |
||||
case reflect.Map: |
||||
if rv.IsNil() { |
||||
return |
||||
} |
||||
enc.eTable(key, rv) |
||||
case reflect.Ptr: |
||||
if rv.IsNil() { |
||||
return |
||||
} |
||||
enc.encode(key, rv.Elem()) |
||||
case reflect.Struct: |
||||
enc.eTable(key, rv) |
||||
default: |
||||
panic(e("unsupported type for key '%s': %s", key, k)) |
||||
} |
||||
} |
||||
|
||||
// eElement encodes any value that can be an array element (primitives and
|
||||
// arrays).
|
||||
func (enc *Encoder) eElement(rv reflect.Value) { |
||||
switch v := rv.Interface().(type) { |
||||
case time.Time: |
||||
// Special case time.Time as a primitive. Has to come before
|
||||
// TextMarshaler below because time.Time implements
|
||||
// encoding.TextMarshaler, but we need to always use UTC.
|
||||
enc.wf(v.UTC().Format("2006-01-02T15:04:05Z")) |
||||
return |
||||
case TextMarshaler: |
||||
// Special case. Use text marshaler if it's available for this value.
|
||||
if s, err := v.MarshalText(); err != nil { |
||||
encPanic(err) |
||||
} else { |
||||
enc.writeQuoted(string(s)) |
||||
} |
||||
return |
||||
} |
||||
switch rv.Kind() { |
||||
case reflect.Bool: |
||||
enc.wf(strconv.FormatBool(rv.Bool())) |
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, |
||||
reflect.Int64: |
||||
enc.wf(strconv.FormatInt(rv.Int(), 10)) |
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, |
||||
reflect.Uint32, reflect.Uint64: |
||||
enc.wf(strconv.FormatUint(rv.Uint(), 10)) |
||||
case reflect.Float32: |
||||
enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 32))) |
||||
case reflect.Float64: |
||||
enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 64))) |
||||
case reflect.Array, reflect.Slice: |
||||
enc.eArrayOrSliceElement(rv) |
||||
case reflect.Interface: |
||||
enc.eElement(rv.Elem()) |
||||
case reflect.String: |
||||
enc.writeQuoted(rv.String()) |
||||
default: |
||||
panic(e("unexpected primitive type: %s", rv.Kind())) |
||||
} |
||||
} |
||||
|
||||
// By the TOML spec, all floats must have a decimal with at least one
|
||||
// number on either side.
|
||||
func floatAddDecimal(fstr string) string { |
||||
if !strings.Contains(fstr, ".") { |
||||
return fstr + ".0" |
||||
} |
||||
return fstr |
||||
} |
||||
|
||||
func (enc *Encoder) writeQuoted(s string) { |
||||
enc.wf("\"%s\"", quotedReplacer.Replace(s)) |
||||
} |
||||
|
||||
func (enc *Encoder) eArrayOrSliceElement(rv reflect.Value) { |
||||
length := rv.Len() |
||||
enc.wf("[") |
||||
for i := 0; i < length; i++ { |
||||
elem := rv.Index(i) |
||||
enc.eElement(elem) |
||||
if i != length-1 { |
||||
enc.wf(", ") |
||||
} |
||||
} |
||||
enc.wf("]") |
||||
} |
||||
|
||||
func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) { |
||||
if len(key) == 0 { |
||||
encPanic(errNoKey) |
||||
} |
||||
for i := 0; i < rv.Len(); i++ { |
||||
trv := rv.Index(i) |
||||
if isNil(trv) { |
||||
continue |
||||
} |
||||
panicIfInvalidKey(key) |
||||
enc.newline() |
||||
enc.wf("%s[[%s]]", enc.indentStr(key), key.maybeQuotedAll()) |
||||
enc.newline() |
||||
enc.eMapOrStruct(key, trv) |
||||
} |
||||
} |
||||
|
||||
func (enc *Encoder) eTable(key Key, rv reflect.Value) { |
||||
panicIfInvalidKey(key) |
||||
if len(key) == 1 { |
||||
// Output an extra newline between top-level tables.
|
||||
// (The newline isn't written if nothing else has been written though.)
|
||||
enc.newline() |
||||
} |
||||
if len(key) > 0 { |
||||
enc.wf("%s[%s]", enc.indentStr(key), key.maybeQuotedAll()) |
||||
enc.newline() |
||||
} |
||||
enc.eMapOrStruct(key, rv) |
||||
} |
||||
|
||||
func (enc *Encoder) eMapOrStruct(key Key, rv reflect.Value) { |
||||
switch rv := eindirect(rv); rv.Kind() { |
||||
case reflect.Map: |
||||
enc.eMap(key, rv) |
||||
case reflect.Struct: |
||||
enc.eStruct(key, rv) |
||||
default: |
||||
panic("eTable: unhandled reflect.Value Kind: " + rv.Kind().String()) |
||||
} |
||||
} |
||||
|
||||
func (enc *Encoder) eMap(key Key, rv reflect.Value) { |
||||
rt := rv.Type() |
||||
if rt.Key().Kind() != reflect.String { |
||||
encPanic(errNonString) |
||||
} |
||||
|
||||
// Sort keys so that we have deterministic output. And write keys directly
|
||||
// underneath this key first, before writing sub-structs or sub-maps.
|
||||
var mapKeysDirect, mapKeysSub []string |
||||
for _, mapKey := range rv.MapKeys() { |
||||
k := mapKey.String() |
||||
if typeIsHash(tomlTypeOfGo(rv.MapIndex(mapKey))) { |
||||
mapKeysSub = append(mapKeysSub, k) |
||||
} else { |
||||
mapKeysDirect = append(mapKeysDirect, k) |
||||
} |
||||
} |
||||
|
||||
var writeMapKeys = func(mapKeys []string) { |
||||
sort.Strings(mapKeys) |
||||
for _, mapKey := range mapKeys { |
||||
mrv := rv.MapIndex(reflect.ValueOf(mapKey)) |
||||
if isNil(mrv) { |
||||
// Don't write anything for nil fields.
|
||||
continue |
||||
} |
||||
enc.encode(key.add(mapKey), mrv) |
||||
} |
||||
} |
||||
writeMapKeys(mapKeysDirect) |
||||
writeMapKeys(mapKeysSub) |
||||
} |
||||
|
||||
func (enc *Encoder) eStruct(key Key, rv reflect.Value) { |
||||
// Write keys for fields directly under this key first, because if we write
|
||||
// a field that creates a new table, then all keys under it will be in that
|
||||
// table (not the one we're writing here).
|
||||
rt := rv.Type() |
||||
var fieldsDirect, fieldsSub [][]int |
||||
var addFields func(rt reflect.Type, rv reflect.Value, start []int) |
||||
addFields = func(rt reflect.Type, rv reflect.Value, start []int) { |
||||
for i := 0; i < rt.NumField(); i++ { |
||||
f := rt.Field(i) |
||||
// skip unexported fields
|
||||
if f.PkgPath != "" && !f.Anonymous { |
||||
continue |
||||
} |
||||
frv := rv.Field(i) |
||||
if f.Anonymous { |
||||
t := f.Type |
||||
switch t.Kind() { |
||||
case reflect.Struct: |
||||
// Treat anonymous struct fields with
|
||||
// tag names as though they are not
|
||||
// anonymous, like encoding/json does.
|
||||
if getOptions(f.Tag).name == "" { |
||||
addFields(t, frv, f.Index) |
||||
continue |
||||
} |
||||
case reflect.Ptr: |
||||
if t.Elem().Kind() == reflect.Struct && |
||||
getOptions(f.Tag).name == "" { |
||||
if !frv.IsNil() { |
||||
addFields(t.Elem(), frv.Elem(), f.Index) |
||||
} |
||||
continue |
||||
} |
||||
// Fall through to the normal field encoding logic below
|
||||
// for non-struct anonymous fields.
|
||||
} |
||||
} |
||||
|
||||
if typeIsHash(tomlTypeOfGo(frv)) { |
||||
fieldsSub = append(fieldsSub, append(start, f.Index...)) |
||||
} else { |
||||
fieldsDirect = append(fieldsDirect, append(start, f.Index...)) |
||||
} |
||||
} |
||||
} |
||||
addFields(rt, rv, nil) |
||||
|
||||
var writeFields = func(fields [][]int) { |
||||
for _, fieldIndex := range fields { |
||||
sft := rt.FieldByIndex(fieldIndex) |
||||
sf := rv.FieldByIndex(fieldIndex) |
||||
if isNil(sf) { |
||||
// Don't write anything for nil fields.
|
||||
continue |
||||
} |
||||
|
||||
opts := getOptions(sft.Tag) |
||||
if opts.skip { |
||||
continue |
||||
} |
||||
keyName := sft.Name |
||||
if opts.name != "" { |
||||
keyName = opts.name |
||||
} |
||||
if opts.omitempty && isEmpty(sf) { |
||||
continue |
||||
} |
||||
if opts.omitzero && isZero(sf) { |
||||
continue |
||||
} |
||||
|
||||
enc.encode(key.add(keyName), sf) |
||||
} |
||||
} |
||||
writeFields(fieldsDirect) |
||||
writeFields(fieldsSub) |
||||
} |
||||
|
||||
// tomlTypeName returns the TOML type name of the Go value's type. It is
|
||||
// used to determine whether the types of array elements are mixed (which is
|
||||
// forbidden). If the Go value is nil, then it is illegal for it to be an array
|
||||
// element, and valueIsNil is returned as true.
|
||||
|
||||
// Returns the TOML type of a Go value. The type may be `nil`, which means
|
||||
// no concrete TOML type could be found.
|
||||
func tomlTypeOfGo(rv reflect.Value) tomlType { |
||||
if isNil(rv) || !rv.IsValid() { |
||||
return nil |
||||
} |
||||
switch rv.Kind() { |
||||
case reflect.Bool: |
||||
return tomlBool |
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, |
||||
reflect.Int64, |
||||
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, |
||||
reflect.Uint64: |
||||
return tomlInteger |
||||
case reflect.Float32, reflect.Float64: |
||||
return tomlFloat |
||||
case reflect.Array, reflect.Slice: |
||||
if typeEqual(tomlHash, tomlArrayType(rv)) { |
||||
return tomlArrayHash |
||||
} |
||||
return tomlArray |
||||
case reflect.Ptr, reflect.Interface: |
||||
return tomlTypeOfGo(rv.Elem()) |
||||
case reflect.String: |
||||
return tomlString |
||||
case reflect.Map: |
||||
return tomlHash |
||||
case reflect.Struct: |
||||
switch rv.Interface().(type) { |
||||
case time.Time: |
||||
return tomlDatetime |
||||
case TextMarshaler: |
||||
return tomlString |
||||
default: |
||||
return tomlHash |
||||
} |
||||
default: |
||||
panic("unexpected reflect.Kind: " + rv.Kind().String()) |
||||
} |
||||
} |
||||
|
||||
// tomlArrayType returns the element type of a TOML array. The type returned
|
||||
// may be nil if it cannot be determined (e.g., a nil slice or a zero length
|
||||
// slize). This function may also panic if it finds a type that cannot be
|
||||
// expressed in TOML (such as nil elements, heterogeneous arrays or directly
|
||||
// nested arrays of tables).
|
||||
func tomlArrayType(rv reflect.Value) tomlType { |
||||
if isNil(rv) || !rv.IsValid() || rv.Len() == 0 { |
||||
return nil |
||||
} |
||||
firstType := tomlTypeOfGo(rv.Index(0)) |
||||
if firstType == nil { |
||||
encPanic(errArrayNilElement) |
||||
} |
||||
|
||||
rvlen := rv.Len() |
||||
for i := 1; i < rvlen; i++ { |
||||
elem := rv.Index(i) |
||||
switch elemType := tomlTypeOfGo(elem); { |
||||
case elemType == nil: |
||||
encPanic(errArrayNilElement) |
||||
case !typeEqual(firstType, elemType): |
||||
encPanic(errArrayMixedElementTypes) |
||||
} |
||||
} |
||||
// If we have a nested array, then we must make sure that the nested
|
||||
// array contains ONLY primitives.
|
||||
// This checks arbitrarily nested arrays.
|
||||
if typeEqual(firstType, tomlArray) || typeEqual(firstType, tomlArrayHash) { |
||||
nest := tomlArrayType(eindirect(rv.Index(0))) |
||||
if typeEqual(nest, tomlHash) || typeEqual(nest, tomlArrayHash) { |
||||
encPanic(errArrayNoTable) |
||||
} |
||||
} |
||||
return firstType |
||||
} |
||||
|
||||
type tagOptions struct { |
||||
skip bool // "-"
|
||||
name string |
||||
omitempty bool |
||||
omitzero bool |
||||
} |
||||
|
||||
func getOptions(tag reflect.StructTag) tagOptions { |
||||
t := tag.Get("toml") |
||||
if t == "-" { |
||||
return tagOptions{skip: true} |
||||
} |
||||
var opts tagOptions |
||||
parts := strings.Split(t, ",") |
||||
opts.name = parts[0] |
||||
for _, s := range parts[1:] { |
||||
switch s { |
||||
case "omitempty": |
||||
opts.omitempty = true |
||||
case "omitzero": |
||||
opts.omitzero = true |
||||
} |
||||
} |
||||
return opts |
||||
} |
||||
|
||||
func isZero(rv reflect.Value) bool { |
||||
switch rv.Kind() { |
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: |
||||
return rv.Int() == 0 |
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: |
||||
return rv.Uint() == 0 |
||||
case reflect.Float32, reflect.Float64: |
||||
return rv.Float() == 0.0 |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func isEmpty(rv reflect.Value) bool { |
||||
switch rv.Kind() { |
||||
case reflect.Array, reflect.Slice, reflect.Map, reflect.String: |
||||
return rv.Len() == 0 |
||||
case reflect.Bool: |
||||
return !rv.Bool() |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func (enc *Encoder) newline() { |
||||
if enc.hasWritten { |
||||
enc.wf("\n") |
||||
} |
||||
} |
||||
|
||||
func (enc *Encoder) keyEqElement(key Key, val reflect.Value) { |
||||
if len(key) == 0 { |
||||
encPanic(errNoKey) |
||||
} |
||||
panicIfInvalidKey(key) |
||||
enc.wf("%s%s = ", enc.indentStr(key), key.maybeQuoted(len(key)-1)) |
||||
enc.eElement(val) |
||||
enc.newline() |
||||
} |
||||
|
||||
func (enc *Encoder) wf(format string, v ...interface{}) { |
||||
if _, err := fmt.Fprintf(enc.w, format, v...); err != nil { |
||||
encPanic(err) |
||||
} |
||||
enc.hasWritten = true |
||||
} |
||||
|
||||
func (enc *Encoder) indentStr(key Key) string { |
||||
return strings.Repeat(enc.Indent, len(key)-1) |
||||
} |
||||
|
||||
func encPanic(err error) { |
||||
panic(tomlEncodeError{err}) |
||||
} |
||||
|
||||
func eindirect(v reflect.Value) reflect.Value { |
||||
switch v.Kind() { |
||||
case reflect.Ptr, reflect.Interface: |
||||
return eindirect(v.Elem()) |
||||
default: |
||||
return v |
||||
} |
||||
} |
||||
|
||||
func isNil(rv reflect.Value) bool { |
||||
switch rv.Kind() { |
||||
case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: |
||||
return rv.IsNil() |
||||
default: |
||||
return false |
||||
} |
||||
} |
||||
|
||||
func panicIfInvalidKey(key Key) { |
||||
for _, k := range key { |
||||
if len(k) == 0 { |
||||
encPanic(e("Key '%s' is not a valid table name. Key names "+ |
||||
"cannot be empty.", key.maybeQuotedAll())) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func isValidKeyName(s string) bool { |
||||
return len(s) != 0 |
||||
} |
||||
@ -0,0 +1,19 @@ |
||||
// +build go1.2
|
||||
|
||||
package toml |
||||
|
||||
// In order to support Go 1.1, we define our own TextMarshaler and
|
||||
// TextUnmarshaler types. For Go 1.2+, we just alias them with the
|
||||
// standard library interfaces.
|
||||
|
||||
import ( |
||||
"encoding" |
||||
) |
||||
|
||||
// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here
|
||||
// so that Go 1.1 can be supported.
|
||||
type TextMarshaler encoding.TextMarshaler |
||||
|
||||
// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined
|
||||
// here so that Go 1.1 can be supported.
|
||||
type TextUnmarshaler encoding.TextUnmarshaler |
||||
@ -0,0 +1,18 @@ |
||||
// +build !go1.2
|
||||
|
||||
package toml |
||||
|
||||
// These interfaces were introduced in Go 1.2, so we add them manually when
|
||||
// compiling for Go 1.1.
|
||||
|
||||
// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here
|
||||
// so that Go 1.1 can be supported.
|
||||
type TextMarshaler interface { |
||||
MarshalText() (text []byte, err error) |
||||
} |
||||
|
||||
// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined
|
||||
// here so that Go 1.1 can be supported.
|
||||
type TextUnmarshaler interface { |
||||
UnmarshalText(text []byte) error |
||||
} |
||||
@ -0,0 +1,953 @@ |
||||
package toml |
||||
|
||||
import ( |
||||
"fmt" |
||||
"strings" |
||||
"unicode" |
||||
"unicode/utf8" |
||||
) |
||||
|
||||
type itemType int |
||||
|
||||
const ( |
||||
itemError itemType = iota |
||||
itemNIL // used in the parser to indicate no type
|
||||
itemEOF |
||||
itemText |
||||
itemString |
||||
itemRawString |
||||
itemMultilineString |
||||
itemRawMultilineString |
||||
itemBool |
||||
itemInteger |
||||
itemFloat |
||||
itemDatetime |
||||
itemArray // the start of an array
|
||||
itemArrayEnd |
||||
itemTableStart |
||||
itemTableEnd |
||||
itemArrayTableStart |
||||
itemArrayTableEnd |
||||
itemKeyStart |
||||
itemCommentStart |
||||
itemInlineTableStart |
||||
itemInlineTableEnd |
||||
) |
||||
|
||||
const ( |
||||
eof = 0 |
||||
comma = ',' |
||||
tableStart = '[' |
||||
tableEnd = ']' |
||||
arrayTableStart = '[' |
||||
arrayTableEnd = ']' |
||||
tableSep = '.' |
||||
keySep = '=' |
||||
arrayStart = '[' |
||||
arrayEnd = ']' |
||||
commentStart = '#' |
||||
stringStart = '"' |
||||
stringEnd = '"' |
||||
rawStringStart = '\'' |
||||
rawStringEnd = '\'' |
||||
inlineTableStart = '{' |
||||
inlineTableEnd = '}' |
||||
) |
||||
|
||||
type stateFn func(lx *lexer) stateFn |
||||
|
||||
type lexer struct { |
||||
input string |
||||
start int |
||||
pos int |
||||
line int |
||||
state stateFn |
||||
items chan item |
||||
|
||||
// Allow for backing up up to three runes.
|
||||
// This is necessary because TOML contains 3-rune tokens (""" and ''').
|
||||
prevWidths [3]int |
||||
nprev int // how many of prevWidths are in use
|
||||
// If we emit an eof, we can still back up, but it is not OK to call
|
||||
// next again.
|
||||
atEOF bool |
||||
|
||||
// A stack of state functions used to maintain context.
|
||||
// The idea is to reuse parts of the state machine in various places.
|
||||
// For example, values can appear at the top level or within arbitrarily
|
||||
// nested arrays. The last state on the stack is used after a value has
|
||||
// been lexed. Similarly for comments.
|
||||
stack []stateFn |
||||
} |
||||
|
||||
type item struct { |
||||
typ itemType |
||||
val string |
||||
line int |
||||
} |
||||
|
||||
func (lx *lexer) nextItem() item { |
||||
for { |
||||
select { |
||||
case item := <-lx.items: |
||||
return item |
||||
default: |
||||
lx.state = lx.state(lx) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func lex(input string) *lexer { |
||||
lx := &lexer{ |
||||
input: input, |
||||
state: lexTop, |
||||
line: 1, |
||||
items: make(chan item, 10), |
||||
stack: make([]stateFn, 0, 10), |
||||
} |
||||
return lx |
||||
} |
||||
|
||||
func (lx *lexer) push(state stateFn) { |
||||
lx.stack = append(lx.stack, state) |
||||
} |
||||
|
||||
func (lx *lexer) pop() stateFn { |
||||
if len(lx.stack) == 0 { |
||||
return lx.errorf("BUG in lexer: no states to pop") |
||||
} |
||||
last := lx.stack[len(lx.stack)-1] |
||||
lx.stack = lx.stack[0 : len(lx.stack)-1] |
||||
return last |
||||
} |
||||
|
||||
func (lx *lexer) current() string { |
||||
return lx.input[lx.start:lx.pos] |
||||
} |
||||
|
||||
func (lx *lexer) emit(typ itemType) { |
||||
lx.items <- item{typ, lx.current(), lx.line} |
||||
lx.start = lx.pos |
||||
} |
||||
|
||||
func (lx *lexer) emitTrim(typ itemType) { |
||||
lx.items <- item{typ, strings.TrimSpace(lx.current()), lx.line} |
||||
lx.start = lx.pos |
||||
} |
||||
|
||||
func (lx *lexer) next() (r rune) { |
||||
if lx.atEOF { |
||||
panic("next called after EOF") |
||||
} |
||||
if lx.pos >= len(lx.input) { |
||||
lx.atEOF = true |
||||
return eof |
||||
} |
||||
|
||||
if lx.input[lx.pos] == '\n' { |
||||
lx.line++ |
||||
} |
||||
lx.prevWidths[2] = lx.prevWidths[1] |
||||
lx.prevWidths[1] = lx.prevWidths[0] |
||||
if lx.nprev < 3 { |
||||
lx.nprev++ |
||||
} |
||||
r, w := utf8.DecodeRuneInString(lx.input[lx.pos:]) |
||||
lx.prevWidths[0] = w |
||||
lx.pos += w |
||||
return r |
||||
} |
||||
|
||||
// ignore skips over the pending input before this point.
|
||||
func (lx *lexer) ignore() { |
||||
lx.start = lx.pos |
||||
} |
||||
|
||||
// backup steps back one rune. Can be called only twice between calls to next.
|
||||
func (lx *lexer) backup() { |
||||
if lx.atEOF { |
||||
lx.atEOF = false |
||||
return |
||||
} |
||||
if lx.nprev < 1 { |
||||
panic("backed up too far") |
||||
} |
||||
w := lx.prevWidths[0] |
||||
lx.prevWidths[0] = lx.prevWidths[1] |
||||
lx.prevWidths[1] = lx.prevWidths[2] |
||||
lx.nprev-- |
||||
lx.pos -= w |
||||
if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' { |
||||
lx.line-- |
||||
} |
||||
} |
||||
|
||||
// accept consumes the next rune if it's equal to `valid`.
|
||||
func (lx *lexer) accept(valid rune) bool { |
||||
if lx.next() == valid { |
||||
return true |
||||
} |
||||
lx.backup() |
||||
return false |
||||
} |
||||
|
||||
// peek returns but does not consume the next rune in the input.
|
||||
func (lx *lexer) peek() rune { |
||||
r := lx.next() |
||||
lx.backup() |
||||
return r |
||||
} |
||||
|
||||
// skip ignores all input that matches the given predicate.
|
||||
func (lx *lexer) skip(pred func(rune) bool) { |
||||
for { |
||||
r := lx.next() |
||||
if pred(r) { |
||||
continue |
||||
} |
||||
lx.backup() |
||||
lx.ignore() |
||||
return |
||||
} |
||||
} |
||||
|
||||
// errorf stops all lexing by emitting an error and returning `nil`.
|
||||
// Note that any value that is a character is escaped if it's a special
|
||||
// character (newlines, tabs, etc.).
|
||||
func (lx *lexer) errorf(format string, values ...interface{}) stateFn { |
||||
lx.items <- item{ |
||||
itemError, |
||||
fmt.Sprintf(format, values...), |
||||
lx.line, |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// lexTop consumes elements at the top level of TOML data.
|
||||
func lexTop(lx *lexer) stateFn { |
||||
r := lx.next() |
||||
if isWhitespace(r) || isNL(r) { |
||||
return lexSkip(lx, lexTop) |
||||
} |
||||
switch r { |
||||
case commentStart: |
||||
lx.push(lexTop) |
||||
return lexCommentStart |
||||
case tableStart: |
||||
return lexTableStart |
||||
case eof: |
||||
if lx.pos > lx.start { |
||||
return lx.errorf("unexpected EOF") |
||||
} |
||||
lx.emit(itemEOF) |
||||
return nil |
||||
} |
||||
|
||||
// At this point, the only valid item can be a key, so we back up
|
||||
// and let the key lexer do the rest.
|
||||
lx.backup() |
||||
lx.push(lexTopEnd) |
||||
return lexKeyStart |
||||
} |
||||
|
||||
// lexTopEnd is entered whenever a top-level item has been consumed. (A value
|
||||
// or a table.) It must see only whitespace, and will turn back to lexTop
|
||||
// upon a newline. If it sees EOF, it will quit the lexer successfully.
|
||||
func lexTopEnd(lx *lexer) stateFn { |
||||
r := lx.next() |
||||
switch { |
||||
case r == commentStart: |
||||
// a comment will read to a newline for us.
|
||||
lx.push(lexTop) |
||||
return lexCommentStart |
||||
case isWhitespace(r): |
||||
return lexTopEnd |
||||
case isNL(r): |
||||
lx.ignore() |
||||
return lexTop |
||||
case r == eof: |
||||
lx.emit(itemEOF) |
||||
return nil |
||||
} |
||||
return lx.errorf("expected a top-level item to end with a newline, "+ |
||||
"comment, or EOF, but got %q instead", r) |
||||
} |
||||
|
||||
// lexTable lexes the beginning of a table. Namely, it makes sure that
|
||||
// it starts with a character other than '.' and ']'.
|
||||
// It assumes that '[' has already been consumed.
|
||||
// It also handles the case that this is an item in an array of tables.
|
||||
// e.g., '[[name]]'.
|
||||
func lexTableStart(lx *lexer) stateFn { |
||||
if lx.peek() == arrayTableStart { |
||||
lx.next() |
||||
lx.emit(itemArrayTableStart) |
||||
lx.push(lexArrayTableEnd) |
||||
} else { |
||||
lx.emit(itemTableStart) |
||||
lx.push(lexTableEnd) |
||||
} |
||||
return lexTableNameStart |
||||
} |
||||
|
||||
func lexTableEnd(lx *lexer) stateFn { |
||||
lx.emit(itemTableEnd) |
||||
return lexTopEnd |
||||
} |
||||
|
||||
func lexArrayTableEnd(lx *lexer) stateFn { |
||||
if r := lx.next(); r != arrayTableEnd { |
||||
return lx.errorf("expected end of table array name delimiter %q, "+ |
||||
"but got %q instead", arrayTableEnd, r) |
||||
} |
||||
lx.emit(itemArrayTableEnd) |
||||
return lexTopEnd |
||||
} |
||||
|
||||
func lexTableNameStart(lx *lexer) stateFn { |
||||
lx.skip(isWhitespace) |
||||
switch r := lx.peek(); { |
||||
case r == tableEnd || r == eof: |
||||
return lx.errorf("unexpected end of table name " + |
||||
"(table names cannot be empty)") |
||||
case r == tableSep: |
||||
return lx.errorf("unexpected table separator " + |
||||
"(table names cannot be empty)") |
||||
case r == stringStart || r == rawStringStart: |
||||
lx.ignore() |
||||
lx.push(lexTableNameEnd) |
||||
return lexValue // reuse string lexing
|
||||
default: |
||||
return lexBareTableName |
||||
} |
||||
} |
||||
|
||||
// lexBareTableName lexes the name of a table. It assumes that at least one
|
||||
// valid character for the table has already been read.
|
||||
func lexBareTableName(lx *lexer) stateFn { |
||||
r := lx.next() |
||||
if isBareKeyChar(r) { |
||||
return lexBareTableName |
||||
} |
||||
lx.backup() |
||||
lx.emit(itemText) |
||||
return lexTableNameEnd |
||||
} |
||||
|
||||
// lexTableNameEnd reads the end of a piece of a table name, optionally
|
||||
// consuming whitespace.
|
||||
func lexTableNameEnd(lx *lexer) stateFn { |
||||
lx.skip(isWhitespace) |
||||
switch r := lx.next(); { |
||||
case isWhitespace(r): |
||||
return lexTableNameEnd |
||||
case r == tableSep: |
||||
lx.ignore() |
||||
return lexTableNameStart |
||||
case r == tableEnd: |
||||
return lx.pop() |
||||
default: |
||||
return lx.errorf("expected '.' or ']' to end table name, "+ |
||||
"but got %q instead", r) |
||||
} |
||||
} |
||||
|
||||
// lexKeyStart consumes a key name up until the first non-whitespace character.
|
||||
// lexKeyStart will ignore whitespace.
|
||||
func lexKeyStart(lx *lexer) stateFn { |
||||
r := lx.peek() |
||||
switch { |
||||
case r == keySep: |
||||
return lx.errorf("unexpected key separator %q", keySep) |
||||
case isWhitespace(r) || isNL(r): |
||||
lx.next() |
||||
return lexSkip(lx, lexKeyStart) |
||||
case r == stringStart || r == rawStringStart: |
||||
lx.ignore() |
||||
lx.emit(itemKeyStart) |
||||
lx.push(lexKeyEnd) |
||||
return lexValue // reuse string lexing
|
||||
default: |
||||
lx.ignore() |
||||
lx.emit(itemKeyStart) |
||||
return lexBareKey |
||||
} |
||||
} |
||||
|
||||
// lexBareKey consumes the text of a bare key. Assumes that the first character
|
||||
// (which is not whitespace) has not yet been consumed.
|
||||
func lexBareKey(lx *lexer) stateFn { |
||||
switch r := lx.next(); { |
||||
case isBareKeyChar(r): |
||||
return lexBareKey |
||||
case isWhitespace(r): |
||||
lx.backup() |
||||
lx.emit(itemText) |
||||
return lexKeyEnd |
||||
case r == keySep: |
||||
lx.backup() |
||||
lx.emit(itemText) |
||||
return lexKeyEnd |
||||
default: |
||||
return lx.errorf("bare keys cannot contain %q", r) |
||||
} |
||||
} |
||||
|
||||
// lexKeyEnd consumes the end of a key and trims whitespace (up to the key
|
||||
// separator).
|
||||
func lexKeyEnd(lx *lexer) stateFn { |
||||
switch r := lx.next(); { |
||||
case r == keySep: |
||||
return lexSkip(lx, lexValue) |
||||
case isWhitespace(r): |
||||
return lexSkip(lx, lexKeyEnd) |
||||
default: |
||||
return lx.errorf("expected key separator %q, but got %q instead", |
||||
keySep, r) |
||||
} |
||||
} |
||||
|
||||
// lexValue starts the consumption of a value anywhere a value is expected.
|
||||
// lexValue will ignore whitespace.
|
||||
// After a value is lexed, the last state on the next is popped and returned.
|
||||
func lexValue(lx *lexer) stateFn { |
||||
// We allow whitespace to precede a value, but NOT newlines.
|
||||
// In array syntax, the array states are responsible for ignoring newlines.
|
||||
r := lx.next() |
||||
switch { |
||||
case isWhitespace(r): |
||||
return lexSkip(lx, lexValue) |
||||
case isDigit(r): |
||||
lx.backup() // avoid an extra state and use the same as above
|
||||
return lexNumberOrDateStart |
||||
} |
||||
switch r { |
||||
case arrayStart: |
||||
lx.ignore() |
||||
lx.emit(itemArray) |
||||
return lexArrayValue |
||||
case inlineTableStart: |
||||
lx.ignore() |
||||
lx.emit(itemInlineTableStart) |
||||
return lexInlineTableValue |
||||
case stringStart: |
||||
if lx.accept(stringStart) { |
||||
if lx.accept(stringStart) { |
||||
lx.ignore() // Ignore """
|
||||
return lexMultilineString |
||||
} |
||||
lx.backup() |
||||
} |
||||
lx.ignore() // ignore the '"'
|
||||
return lexString |
||||
case rawStringStart: |
||||
if lx.accept(rawStringStart) { |
||||
if lx.accept(rawStringStart) { |
||||
lx.ignore() // Ignore """
|
||||
return lexMultilineRawString |
||||
} |
||||
lx.backup() |
||||
} |
||||
lx.ignore() // ignore the "'"
|
||||
return lexRawString |
||||
case '+', '-': |
||||
return lexNumberStart |
||||
case '.': // special error case, be kind to users
|
||||
return lx.errorf("floats must start with a digit, not '.'") |
||||
} |
||||
if unicode.IsLetter(r) { |
||||
// Be permissive here; lexBool will give a nice error if the
|
||||
// user wrote something like
|
||||
// x = foo
|
||||
// (i.e. not 'true' or 'false' but is something else word-like.)
|
||||
lx.backup() |
||||
return lexBool |
||||
} |
||||
return lx.errorf("expected value but found %q instead", r) |
||||
} |
||||
|
||||
// lexArrayValue consumes one value in an array. It assumes that '[' or ','
|
||||
// have already been consumed. All whitespace and newlines are ignored.
|
||||
func lexArrayValue(lx *lexer) stateFn { |
||||
r := lx.next() |
||||
switch { |
||||
case isWhitespace(r) || isNL(r): |
||||
return lexSkip(lx, lexArrayValue) |
||||
case r == commentStart: |
||||
lx.push(lexArrayValue) |
||||
return lexCommentStart |
||||
case r == comma: |
||||
return lx.errorf("unexpected comma") |
||||
case r == arrayEnd: |
||||
// NOTE(caleb): The spec isn't clear about whether you can have
|
||||
// a trailing comma or not, so we'll allow it.
|
||||
return lexArrayEnd |
||||
} |
||||
|
||||
lx.backup() |
||||
lx.push(lexArrayValueEnd) |
||||
return lexValue |
||||
} |
||||
|
||||
// lexArrayValueEnd consumes everything between the end of an array value and
|
||||
// the next value (or the end of the array): it ignores whitespace and newlines
|
||||
// and expects either a ',' or a ']'.
|
||||
func lexArrayValueEnd(lx *lexer) stateFn { |
||||
r := lx.next() |
||||
switch { |
||||
case isWhitespace(r) || isNL(r): |
||||
return lexSkip(lx, lexArrayValueEnd) |
||||
case r == commentStart: |
||||
lx.push(lexArrayValueEnd) |
||||
return lexCommentStart |
||||
case r == comma: |
||||
lx.ignore() |
||||
return lexArrayValue // move on to the next value
|
||||
case r == arrayEnd: |
||||
return lexArrayEnd |
||||
} |
||||
return lx.errorf( |
||||
"expected a comma or array terminator %q, but got %q instead", |
||||
arrayEnd, r, |
||||
) |
||||
} |
||||
|
||||
// lexArrayEnd finishes the lexing of an array.
|
||||
// It assumes that a ']' has just been consumed.
|
||||
func lexArrayEnd(lx *lexer) stateFn { |
||||
lx.ignore() |
||||
lx.emit(itemArrayEnd) |
||||
return lx.pop() |
||||
} |
||||
|
||||
// lexInlineTableValue consumes one key/value pair in an inline table.
|
||||
// It assumes that '{' or ',' have already been consumed. Whitespace is ignored.
|
||||
func lexInlineTableValue(lx *lexer) stateFn { |
||||
r := lx.next() |
||||
switch { |
||||
case isWhitespace(r): |
||||
return lexSkip(lx, lexInlineTableValue) |
||||
case isNL(r): |
||||
return lx.errorf("newlines not allowed within inline tables") |
||||
case r == commentStart: |
||||
lx.push(lexInlineTableValue) |
||||
return lexCommentStart |
||||
case r == comma: |
||||
return lx.errorf("unexpected comma") |
||||
case r == inlineTableEnd: |
||||
return lexInlineTableEnd |
||||
} |
||||
lx.backup() |
||||
lx.push(lexInlineTableValueEnd) |
||||
return lexKeyStart |
||||
} |
||||
|
||||
// lexInlineTableValueEnd consumes everything between the end of an inline table
|
||||
// key/value pair and the next pair (or the end of the table):
|
||||
// it ignores whitespace and expects either a ',' or a '}'.
|
||||
func lexInlineTableValueEnd(lx *lexer) stateFn { |
||||
r := lx.next() |
||||
switch { |
||||
case isWhitespace(r): |
||||
return lexSkip(lx, lexInlineTableValueEnd) |
||||
case isNL(r): |
||||
return lx.errorf("newlines not allowed within inline tables") |
||||
case r == commentStart: |
||||
lx.push(lexInlineTableValueEnd) |
||||
return lexCommentStart |
||||
case r == comma: |
||||
lx.ignore() |
||||
return lexInlineTableValue |
||||
case r == inlineTableEnd: |
||||
return lexInlineTableEnd |
||||
} |
||||
return lx.errorf("expected a comma or an inline table terminator %q, "+ |
||||
"but got %q instead", inlineTableEnd, r) |
||||
} |
||||
|
||||
// lexInlineTableEnd finishes the lexing of an inline table.
|
||||
// It assumes that a '}' has just been consumed.
|
||||
func lexInlineTableEnd(lx *lexer) stateFn { |
||||
lx.ignore() |
||||
lx.emit(itemInlineTableEnd) |
||||
return lx.pop() |
||||
} |
||||
|
||||
// lexString consumes the inner contents of a string. It assumes that the
|
||||
// beginning '"' has already been consumed and ignored.
|
||||
func lexString(lx *lexer) stateFn { |
||||
r := lx.next() |
||||
switch { |
||||
case r == eof: |
||||
return lx.errorf("unexpected EOF") |
||||
case isNL(r): |
||||
return lx.errorf("strings cannot contain newlines") |
||||
case r == '\\': |
||||
lx.push(lexString) |
||||
return lexStringEscape |
||||
case r == stringEnd: |
||||
lx.backup() |
||||
lx.emit(itemString) |
||||
lx.next() |
||||
lx.ignore() |
||||
return lx.pop() |
||||
} |
||||
return lexString |
||||
} |
||||
|
||||
// lexMultilineString consumes the inner contents of a string. It assumes that
|
||||
// the beginning '"""' has already been consumed and ignored.
|
||||
func lexMultilineString(lx *lexer) stateFn { |
||||
switch lx.next() { |
||||
case eof: |
||||
return lx.errorf("unexpected EOF") |
||||
case '\\': |
||||
return lexMultilineStringEscape |
||||
case stringEnd: |
||||
if lx.accept(stringEnd) { |
||||
if lx.accept(stringEnd) { |
||||
lx.backup() |
||||
lx.backup() |
||||
lx.backup() |
||||
lx.emit(itemMultilineString) |
||||
lx.next() |
||||
lx.next() |
||||
lx.next() |
||||
lx.ignore() |
||||
return lx.pop() |
||||
} |
||||
lx.backup() |
||||
} |
||||
} |
||||
return lexMultilineString |
||||
} |
||||
|
||||
// lexRawString consumes a raw string. Nothing can be escaped in such a string.
|
||||
// It assumes that the beginning "'" has already been consumed and ignored.
|
||||
func lexRawString(lx *lexer) stateFn { |
||||
r := lx.next() |
||||
switch { |
||||
case r == eof: |
||||
return lx.errorf("unexpected EOF") |
||||
case isNL(r): |
||||
return lx.errorf("strings cannot contain newlines") |
||||
case r == rawStringEnd: |
||||
lx.backup() |
||||
lx.emit(itemRawString) |
||||
lx.next() |
||||
lx.ignore() |
||||
return lx.pop() |
||||
} |
||||
return lexRawString |
||||
} |
||||
|
||||
// lexMultilineRawString consumes a raw string. Nothing can be escaped in such
|
||||
// a string. It assumes that the beginning "'''" has already been consumed and
|
||||
// ignored.
|
||||
func lexMultilineRawString(lx *lexer) stateFn { |
||||
switch lx.next() { |
||||
case eof: |
||||
return lx.errorf("unexpected EOF") |
||||
case rawStringEnd: |
||||
if lx.accept(rawStringEnd) { |
||||
if lx.accept(rawStringEnd) { |
||||
lx.backup() |
||||
lx.backup() |
||||
lx.backup() |
||||
lx.emit(itemRawMultilineString) |
||||
lx.next() |
||||
lx.next() |
||||
lx.next() |
||||
lx.ignore() |
||||
return lx.pop() |
||||
} |
||||
lx.backup() |
||||
} |
||||
} |
||||
return lexMultilineRawString |
||||
} |
||||
|
||||
// lexMultilineStringEscape consumes an escaped character. It assumes that the
|
||||
// preceding '\\' has already been consumed.
|
||||
func lexMultilineStringEscape(lx *lexer) stateFn { |
||||
// Handle the special case first:
|
||||
if isNL(lx.next()) { |
||||
return lexMultilineString |
||||
} |
||||
lx.backup() |
||||
lx.push(lexMultilineString) |
||||
return lexStringEscape(lx) |
||||
} |
||||
|
||||
func lexStringEscape(lx *lexer) stateFn { |
||||
r := lx.next() |
||||
switch r { |
||||
case 'b': |
||||
fallthrough |
||||
case 't': |
||||
fallthrough |
||||
case 'n': |
||||
fallthrough |
||||
case 'f': |
||||
fallthrough |
||||
case 'r': |
||||
fallthrough |
||||
case '"': |
||||
fallthrough |
||||
case '\\': |
||||
return lx.pop() |
||||
case 'u': |
||||
return lexShortUnicodeEscape |
||||
case 'U': |
||||
return lexLongUnicodeEscape |
||||
} |
||||
return lx.errorf("invalid escape character %q; only the following "+ |
||||
"escape characters are allowed: "+ |
||||
`\b, \t, \n, \f, \r, \", \\, \uXXXX, and \UXXXXXXXX`, r) |
||||
} |
||||
|
||||
func lexShortUnicodeEscape(lx *lexer) stateFn { |
||||
var r rune |
||||
for i := 0; i < 4; i++ { |
||||
r = lx.next() |
||||
if !isHexadecimal(r) { |
||||
return lx.errorf(`expected four hexadecimal digits after '\u', `+ |
||||
"but got %q instead", lx.current()) |
||||
} |
||||
} |
||||
return lx.pop() |
||||
} |
||||
|
||||
func lexLongUnicodeEscape(lx *lexer) stateFn { |
||||
var r rune |
||||
for i := 0; i < 8; i++ { |
||||
r = lx.next() |
||||
if !isHexadecimal(r) { |
||||
return lx.errorf(`expected eight hexadecimal digits after '\U', `+ |
||||
"but got %q instead", lx.current()) |
||||
} |
||||
} |
||||
return lx.pop() |
||||
} |
||||
|
||||
// lexNumberOrDateStart consumes either an integer, a float, or datetime.
|
||||
func lexNumberOrDateStart(lx *lexer) stateFn { |
||||
r := lx.next() |
||||
if isDigit(r) { |
||||
return lexNumberOrDate |
||||
} |
||||
switch r { |
||||
case '_': |
||||
return lexNumber |
||||
case 'e', 'E': |
||||
return lexFloat |
||||
case '.': |
||||
return lx.errorf("floats must start with a digit, not '.'") |
||||
} |
||||
return lx.errorf("expected a digit but got %q", r) |
||||
} |
||||
|
||||
// lexNumberOrDate consumes either an integer, float or datetime.
|
||||
func lexNumberOrDate(lx *lexer) stateFn { |
||||
r := lx.next() |
||||
if isDigit(r) { |
||||
return lexNumberOrDate |
||||
} |
||||
switch r { |
||||
case '-': |
||||
return lexDatetime |
||||
case '_': |
||||
return lexNumber |
||||
case '.', 'e', 'E': |
||||
return lexFloat |
||||
} |
||||
|
||||
lx.backup() |
||||
lx.emit(itemInteger) |
||||
return lx.pop() |
||||
} |
||||
|
||||
// lexDatetime consumes a Datetime, to a first approximation.
|
||||
// The parser validates that it matches one of the accepted formats.
|
||||
func lexDatetime(lx *lexer) stateFn { |
||||
r := lx.next() |
||||
if isDigit(r) { |
||||
return lexDatetime |
||||
} |
||||
switch r { |
||||
case '-', 'T', ':', '.', 'Z', '+': |
||||
return lexDatetime |
||||
} |
||||
|
||||
lx.backup() |
||||
lx.emit(itemDatetime) |
||||
return lx.pop() |
||||
} |
||||
|
||||
// lexNumberStart consumes either an integer or a float. It assumes that a sign
|
||||
// has already been read, but that *no* digits have been consumed.
|
||||
// lexNumberStart will move to the appropriate integer or float states.
|
||||
func lexNumberStart(lx *lexer) stateFn { |
||||
// We MUST see a digit. Even floats have to start with a digit.
|
||||
r := lx.next() |
||||
if !isDigit(r) { |
||||
if r == '.' { |
||||
return lx.errorf("floats must start with a digit, not '.'") |
||||
} |
||||
return lx.errorf("expected a digit but got %q", r) |
||||
} |
||||
return lexNumber |
||||
} |
||||
|
||||
// lexNumber consumes an integer or a float after seeing the first digit.
|
||||
func lexNumber(lx *lexer) stateFn { |
||||
r := lx.next() |
||||
if isDigit(r) { |
||||
return lexNumber |
||||
} |
||||
switch r { |
||||
case '_': |
||||
return lexNumber |
||||
case '.', 'e', 'E': |
||||
return lexFloat |
||||
} |
||||
|
||||
lx.backup() |
||||
lx.emit(itemInteger) |
||||
return lx.pop() |
||||
} |
||||
|
||||
// lexFloat consumes the elements of a float. It allows any sequence of
|
||||
// float-like characters, so floats emitted by the lexer are only a first
|
||||
// approximation and must be validated by the parser.
|
||||
func lexFloat(lx *lexer) stateFn { |
||||
r := lx.next() |
||||
if isDigit(r) { |
||||
return lexFloat |
||||
} |
||||
switch r { |
||||
case '_', '.', '-', '+', 'e', 'E': |
||||
return lexFloat |
||||
} |
||||
|
||||
lx.backup() |
||||
lx.emit(itemFloat) |
||||
return lx.pop() |
||||
} |
||||
|
||||
// lexBool consumes a bool string: 'true' or 'false.
|
||||
func lexBool(lx *lexer) stateFn { |
||||
var rs []rune |
||||
for { |
||||
r := lx.next() |
||||
if !unicode.IsLetter(r) { |
||||
lx.backup() |
||||
break |
||||
} |
||||
rs = append(rs, r) |
||||
} |
||||
s := string(rs) |
||||
switch s { |
||||
case "true", "false": |
||||
lx.emit(itemBool) |
||||
return lx.pop() |
||||
} |
||||
return lx.errorf("expected value but found %q instead", s) |
||||
} |
||||
|
||||
// lexCommentStart begins the lexing of a comment. It will emit
|
||||
// itemCommentStart and consume no characters, passing control to lexComment.
|
||||
func lexCommentStart(lx *lexer) stateFn { |
||||
lx.ignore() |
||||
lx.emit(itemCommentStart) |
||||
return lexComment |
||||
} |
||||
|
||||
// lexComment lexes an entire comment. It assumes that '#' has been consumed.
|
||||
// It will consume *up to* the first newline character, and pass control
|
||||
// back to the last state on the stack.
|
||||
func lexComment(lx *lexer) stateFn { |
||||
r := lx.peek() |
||||
if isNL(r) || r == eof { |
||||
lx.emit(itemText) |
||||
return lx.pop() |
||||
} |
||||
lx.next() |
||||
return lexComment |
||||
} |
||||
|
||||
// lexSkip ignores all slurped input and moves on to the next state.
|
||||
func lexSkip(lx *lexer, nextState stateFn) stateFn { |
||||
return func(lx *lexer) stateFn { |
||||
lx.ignore() |
||||
return nextState |
||||
} |
||||
} |
||||
|
||||
// isWhitespace returns true if `r` is a whitespace character according
|
||||
// to the spec.
|
||||
func isWhitespace(r rune) bool { |
||||
return r == '\t' || r == ' ' |
||||
} |
||||
|
||||
func isNL(r rune) bool { |
||||
return r == '\n' || r == '\r' |
||||
} |
||||
|
||||
func isDigit(r rune) bool { |
||||
return r >= '0' && r <= '9' |
||||
} |
||||
|
||||
func isHexadecimal(r rune) bool { |
||||
return (r >= '0' && r <= '9') || |
||||
(r >= 'a' && r <= 'f') || |
||||
(r >= 'A' && r <= 'F') |
||||
} |
||||
|
||||
func isBareKeyChar(r rune) bool { |
||||
return (r >= 'A' && r <= 'Z') || |
||||
(r >= 'a' && r <= 'z') || |
||||
(r >= '0' && r <= '9') || |
||||
r == '_' || |
||||
r == '-' |
||||
} |
||||
|
||||
func (itype itemType) String() string { |
||||
switch itype { |
||||
case itemError: |
||||
return "Error" |
||||
case itemNIL: |
||||
return "NIL" |
||||
case itemEOF: |
||||
return "EOF" |
||||
case itemText: |
||||
return "Text" |
||||
case itemString, itemRawString, itemMultilineString, itemRawMultilineString: |
||||
return "String" |
||||
case itemBool: |
||||
return "Bool" |
||||
case itemInteger: |
||||
return "Integer" |
||||
case itemFloat: |
||||
return "Float" |
||||
case itemDatetime: |
||||
return "DateTime" |
||||
case itemTableStart: |
||||
return "TableStart" |
||||
case itemTableEnd: |
||||
return "TableEnd" |
||||
case itemKeyStart: |
||||
return "KeyStart" |
||||
case itemArray: |
||||
return "Array" |
||||
case itemArrayEnd: |
||||
return "ArrayEnd" |
||||
case itemCommentStart: |
||||
return "CommentStart" |
||||
} |
||||
panic(fmt.Sprintf("BUG: Unknown type '%d'.", int(itype))) |
||||
} |
||||
|
||||
func (item item) String() string { |
||||
return fmt.Sprintf("(%s, %s)", item.typ.String(), item.val) |
||||
} |
||||
@ -0,0 +1,592 @@ |
||||
package toml |
||||
|
||||
import ( |
||||
"fmt" |
||||
"strconv" |
||||
"strings" |
||||
"time" |
||||
"unicode" |
||||
"unicode/utf8" |
||||
) |
||||
|
||||
type parser struct { |
||||
mapping map[string]interface{} |
||||
types map[string]tomlType |
||||
lx *lexer |
||||
|
||||
// A list of keys in the order that they appear in the TOML data.
|
||||
ordered []Key |
||||
|
||||
// the full key for the current hash in scope
|
||||
context Key |
||||
|
||||
// the base key name for everything except hashes
|
||||
currentKey string |
||||
|
||||
// rough approximation of line number
|
||||
approxLine int |
||||
|
||||
// A map of 'key.group.names' to whether they were created implicitly.
|
||||
implicits map[string]bool |
||||
} |
||||
|
||||
type parseError string |
||||
|
||||
func (pe parseError) Error() string { |
||||
return string(pe) |
||||
} |
||||
|
||||
func parse(data string) (p *parser, err error) { |
||||
defer func() { |
||||
if r := recover(); r != nil { |
||||
var ok bool |
||||
if err, ok = r.(parseError); ok { |
||||
return |
||||
} |
||||
panic(r) |
||||
} |
||||
}() |
||||
|
||||
p = &parser{ |
||||
mapping: make(map[string]interface{}), |
||||
types: make(map[string]tomlType), |
||||
lx: lex(data), |
||||
ordered: make([]Key, 0), |
||||
implicits: make(map[string]bool), |
||||
} |
||||
for { |
||||
item := p.next() |
||||
if item.typ == itemEOF { |
||||
break |
||||
} |
||||
p.topLevel(item) |
||||
} |
||||
|
||||
return p, nil |
||||
} |
||||
|
||||
func (p *parser) panicf(format string, v ...interface{}) { |
||||
msg := fmt.Sprintf("Near line %d (last key parsed '%s'): %s", |
||||
p.approxLine, p.current(), fmt.Sprintf(format, v...)) |
||||
panic(parseError(msg)) |
||||
} |
||||
|
||||
func (p *parser) next() item { |
||||
it := p.lx.nextItem() |
||||
if it.typ == itemError { |
||||
p.panicf("%s", it.val) |
||||
} |
||||
return it |
||||
} |
||||
|
||||
func (p *parser) bug(format string, v ...interface{}) { |
||||
panic(fmt.Sprintf("BUG: "+format+"\n\n", v...)) |
||||
} |
||||
|
||||
func (p *parser) expect(typ itemType) item { |
||||
it := p.next() |
||||
p.assertEqual(typ, it.typ) |
||||
return it |
||||
} |
||||
|
||||
func (p *parser) assertEqual(expected, got itemType) { |
||||
if expected != got { |
||||
p.bug("Expected '%s' but got '%s'.", expected, got) |
||||
} |
||||
} |
||||
|
||||
func (p *parser) topLevel(item item) { |
||||
switch item.typ { |
||||
case itemCommentStart: |
||||
p.approxLine = item.line |
||||
p.expect(itemText) |
||||
case itemTableStart: |
||||
kg := p.next() |
||||
p.approxLine = kg.line |
||||
|
||||
var key Key |
||||
for ; kg.typ != itemTableEnd && kg.typ != itemEOF; kg = p.next() { |
||||
key = append(key, p.keyString(kg)) |
||||
} |
||||
p.assertEqual(itemTableEnd, kg.typ) |
||||
|
||||
p.establishContext(key, false) |
||||
p.setType("", tomlHash) |
||||
p.ordered = append(p.ordered, key) |
||||
case itemArrayTableStart: |
||||
kg := p.next() |
||||
p.approxLine = kg.line |
||||
|
||||
var key Key |
||||
for ; kg.typ != itemArrayTableEnd && kg.typ != itemEOF; kg = p.next() { |
||||
key = append(key, p.keyString(kg)) |
||||
} |
||||
p.assertEqual(itemArrayTableEnd, kg.typ) |
||||
|
||||
p.establishContext(key, true) |
||||
p.setType("", tomlArrayHash) |
||||
p.ordered = append(p.ordered, key) |
||||
case itemKeyStart: |
||||
kname := p.next() |
||||
p.approxLine = kname.line |
||||
p.currentKey = p.keyString(kname) |
||||
|
||||
val, typ := p.value(p.next()) |
||||
p.setValue(p.currentKey, val) |
||||
p.setType(p.currentKey, typ) |
||||
p.ordered = append(p.ordered, p.context.add(p.currentKey)) |
||||
p.currentKey = "" |
||||
default: |
||||
p.bug("Unexpected type at top level: %s", item.typ) |
||||
} |
||||
} |
||||
|
||||
// Gets a string for a key (or part of a key in a table name).
|
||||
func (p *parser) keyString(it item) string { |
||||
switch it.typ { |
||||
case itemText: |
||||
return it.val |
||||
case itemString, itemMultilineString, |
||||
itemRawString, itemRawMultilineString: |
||||
s, _ := p.value(it) |
||||
return s.(string) |
||||
default: |
||||
p.bug("Unexpected key type: %s", it.typ) |
||||
panic("unreachable") |
||||
} |
||||
} |
||||
|
||||
// value translates an expected value from the lexer into a Go value wrapped
|
||||
// as an empty interface.
|
||||
func (p *parser) value(it item) (interface{}, tomlType) { |
||||
switch it.typ { |
||||
case itemString: |
||||
return p.replaceEscapes(it.val), p.typeOfPrimitive(it) |
||||
case itemMultilineString: |
||||
trimmed := stripFirstNewline(stripEscapedWhitespace(it.val)) |
||||
return p.replaceEscapes(trimmed), p.typeOfPrimitive(it) |
||||
case itemRawString: |
||||
return it.val, p.typeOfPrimitive(it) |
||||
case itemRawMultilineString: |
||||
return stripFirstNewline(it.val), p.typeOfPrimitive(it) |
||||
case itemBool: |
||||
switch it.val { |
||||
case "true": |
||||
return true, p.typeOfPrimitive(it) |
||||
case "false": |
||||
return false, p.typeOfPrimitive(it) |
||||
} |
||||
p.bug("Expected boolean value, but got '%s'.", it.val) |
||||
case itemInteger: |
||||
if !numUnderscoresOK(it.val) { |
||||
p.panicf("Invalid integer %q: underscores must be surrounded by digits", |
||||
it.val) |
||||
} |
||||
val := strings.Replace(it.val, "_", "", -1) |
||||
num, err := strconv.ParseInt(val, 10, 64) |
||||
if err != nil { |
||||
// Distinguish integer values. Normally, it'd be a bug if the lexer
|
||||
// provides an invalid integer, but it's possible that the number is
|
||||
// out of range of valid values (which the lexer cannot determine).
|
||||
// So mark the former as a bug but the latter as a legitimate user
|
||||
// error.
|
||||
if e, ok := err.(*strconv.NumError); ok && |
||||
e.Err == strconv.ErrRange { |
||||
|
||||
p.panicf("Integer '%s' is out of the range of 64-bit "+ |
||||
"signed integers.", it.val) |
||||
} else { |
||||
p.bug("Expected integer value, but got '%s'.", it.val) |
||||
} |
||||
} |
||||
return num, p.typeOfPrimitive(it) |
||||
case itemFloat: |
||||
parts := strings.FieldsFunc(it.val, func(r rune) bool { |
||||
switch r { |
||||
case '.', 'e', 'E': |
||||
return true |
||||
} |
||||
return false |
||||
}) |
||||
for _, part := range parts { |
||||
if !numUnderscoresOK(part) { |
||||
p.panicf("Invalid float %q: underscores must be "+ |
||||
"surrounded by digits", it.val) |
||||
} |
||||
} |
||||
if !numPeriodsOK(it.val) { |
||||
// As a special case, numbers like '123.' or '1.e2',
|
||||
// which are valid as far as Go/strconv are concerned,
|
||||
// must be rejected because TOML says that a fractional
|
||||
// part consists of '.' followed by 1+ digits.
|
||||
p.panicf("Invalid float %q: '.' must be followed "+ |
||||
"by one or more digits", it.val) |
||||
} |
||||
val := strings.Replace(it.val, "_", "", -1) |
||||
num, err := strconv.ParseFloat(val, 64) |
||||
if err != nil { |
||||
if e, ok := err.(*strconv.NumError); ok && |
||||
e.Err == strconv.ErrRange { |
||||
|
||||
p.panicf("Float '%s' is out of the range of 64-bit "+ |
||||
"IEEE-754 floating-point numbers.", it.val) |
||||
} else { |
||||
p.panicf("Invalid float value: %q", it.val) |
||||
} |
||||
} |
||||
return num, p.typeOfPrimitive(it) |
||||
case itemDatetime: |
||||
var t time.Time |
||||
var ok bool |
||||
var err error |
||||
for _, format := range []string{ |
||||
"2006-01-02T15:04:05Z07:00", |
||||
"2006-01-02T15:04:05", |
||||
"2006-01-02", |
||||
} { |
||||
t, err = time.ParseInLocation(format, it.val, time.Local) |
||||
if err == nil { |
||||
ok = true |
||||
break |
||||
} |
||||
} |
||||
if !ok { |
||||
p.panicf("Invalid TOML Datetime: %q.", it.val) |
||||
} |
||||
return t, p.typeOfPrimitive(it) |
||||
case itemArray: |
||||
array := make([]interface{}, 0) |
||||
types := make([]tomlType, 0) |
||||
|
||||
for it = p.next(); it.typ != itemArrayEnd; it = p.next() { |
||||
if it.typ == itemCommentStart { |
||||
p.expect(itemText) |
||||
continue |
||||
} |
||||
|
||||
val, typ := p.value(it) |
||||
array = append(array, val) |
||||
types = append(types, typ) |
||||
} |
||||
return array, p.typeOfArray(types) |
||||
case itemInlineTableStart: |
||||
var ( |
||||
hash = make(map[string]interface{}) |
||||
outerContext = p.context |
||||
outerKey = p.currentKey |
||||
) |
||||
|
||||
p.context = append(p.context, p.currentKey) |
||||
p.currentKey = "" |
||||
for it := p.next(); it.typ != itemInlineTableEnd; it = p.next() { |
||||
if it.typ != itemKeyStart { |
||||
p.bug("Expected key start but instead found %q, around line %d", |
||||
it.val, p.approxLine) |
||||
} |
||||
if it.typ == itemCommentStart { |
||||
p.expect(itemText) |
||||
continue |
||||
} |
||||
|
||||
// retrieve key
|
||||
k := p.next() |
||||
p.approxLine = k.line |
||||
kname := p.keyString(k) |
||||
|
||||
// retrieve value
|
||||
p.currentKey = kname |
||||
val, typ := p.value(p.next()) |
||||
// make sure we keep metadata up to date
|
||||
p.setType(kname, typ) |
||||
p.ordered = append(p.ordered, p.context.add(p.currentKey)) |
||||
hash[kname] = val |
||||
} |
||||
p.context = outerContext |
||||
p.currentKey = outerKey |
||||
return hash, tomlHash |
||||
} |
||||
p.bug("Unexpected value type: %s", it.typ) |
||||
panic("unreachable") |
||||
} |
||||
|
||||
// numUnderscoresOK checks whether each underscore in s is surrounded by
|
||||
// characters that are not underscores.
|
||||
func numUnderscoresOK(s string) bool { |
||||
accept := false |
||||
for _, r := range s { |
||||
if r == '_' { |
||||
if !accept { |
||||
return false |
||||
} |
||||
accept = false |
||||
continue |
||||
} |
||||
accept = true |
||||
} |
||||
return accept |
||||
} |
||||
|
||||
// numPeriodsOK checks whether every period in s is followed by a digit.
|
||||
func numPeriodsOK(s string) bool { |
||||
period := false |
||||
for _, r := range s { |
||||
if period && !isDigit(r) { |
||||
return false |
||||
} |
||||
period = r == '.' |
||||
} |
||||
return !period |
||||
} |
||||
|
||||
// establishContext sets the current context of the parser,
|
||||
// where the context is either a hash or an array of hashes. Which one is
|
||||
// set depends on the value of the `array` parameter.
|
||||
//
|
||||
// Establishing the context also makes sure that the key isn't a duplicate, and
|
||||
// will create implicit hashes automatically.
|
||||
func (p *parser) establishContext(key Key, array bool) { |
||||
var ok bool |
||||
|
||||
// Always start at the top level and drill down for our context.
|
||||
hashContext := p.mapping |
||||
keyContext := make(Key, 0) |
||||
|
||||
// We only need implicit hashes for key[0:-1]
|
||||
for _, k := range key[0 : len(key)-1] { |
||||
_, ok = hashContext[k] |
||||
keyContext = append(keyContext, k) |
||||
|
||||
// No key? Make an implicit hash and move on.
|
||||
if !ok { |
||||
p.addImplicit(keyContext) |
||||
hashContext[k] = make(map[string]interface{}) |
||||
} |
||||
|
||||
// If the hash context is actually an array of tables, then set
|
||||
// the hash context to the last element in that array.
|
||||
//
|
||||
// Otherwise, it better be a table, since this MUST be a key group (by
|
||||
// virtue of it not being the last element in a key).
|
||||
switch t := hashContext[k].(type) { |
||||
case []map[string]interface{}: |
||||
hashContext = t[len(t)-1] |
||||
case map[string]interface{}: |
||||
hashContext = t |
||||
default: |
||||
p.panicf("Key '%s' was already created as a hash.", keyContext) |
||||
} |
||||
} |
||||
|
||||
p.context = keyContext |
||||
if array { |
||||
// If this is the first element for this array, then allocate a new
|
||||
// list of tables for it.
|
||||
k := key[len(key)-1] |
||||
if _, ok := hashContext[k]; !ok { |
||||
hashContext[k] = make([]map[string]interface{}, 0, 5) |
||||
} |
||||
|
||||
// Add a new table. But make sure the key hasn't already been used
|
||||
// for something else.
|
||||
if hash, ok := hashContext[k].([]map[string]interface{}); ok { |
||||
hashContext[k] = append(hash, make(map[string]interface{})) |
||||
} else { |
||||
p.panicf("Key '%s' was already created and cannot be used as "+ |
||||
"an array.", keyContext) |
||||
} |
||||
} else { |
||||
p.setValue(key[len(key)-1], make(map[string]interface{})) |
||||
} |
||||
p.context = append(p.context, key[len(key)-1]) |
||||
} |
||||
|
||||
// setValue sets the given key to the given value in the current context.
|
||||
// It will make sure that the key hasn't already been defined, account for
|
||||
// implicit key groups.
|
||||
func (p *parser) setValue(key string, value interface{}) { |
||||
var tmpHash interface{} |
||||
var ok bool |
||||
|
||||
hash := p.mapping |
||||
keyContext := make(Key, 0) |
||||
for _, k := range p.context { |
||||
keyContext = append(keyContext, k) |
||||
if tmpHash, ok = hash[k]; !ok { |
||||
p.bug("Context for key '%s' has not been established.", keyContext) |
||||
} |
||||
switch t := tmpHash.(type) { |
||||
case []map[string]interface{}: |
||||
// The context is a table of hashes. Pick the most recent table
|
||||
// defined as the current hash.
|
||||
hash = t[len(t)-1] |
||||
case map[string]interface{}: |
||||
hash = t |
||||
default: |
||||
p.bug("Expected hash to have type 'map[string]interface{}', but "+ |
||||
"it has '%T' instead.", tmpHash) |
||||
} |
||||
} |
||||
keyContext = append(keyContext, key) |
||||
|
||||
if _, ok := hash[key]; ok { |
||||
// Typically, if the given key has already been set, then we have
|
||||
// to raise an error since duplicate keys are disallowed. However,
|
||||
// it's possible that a key was previously defined implicitly. In this
|
||||
// case, it is allowed to be redefined concretely. (See the
|
||||
// `tests/valid/implicit-and-explicit-after.toml` test in `toml-test`.)
|
||||
//
|
||||
// But we have to make sure to stop marking it as an implicit. (So that
|
||||
// another redefinition provokes an error.)
|
||||
//
|
||||
// Note that since it has already been defined (as a hash), we don't
|
||||
// want to overwrite it. So our business is done.
|
||||
if p.isImplicit(keyContext) { |
||||
p.removeImplicit(keyContext) |
||||
return |
||||
} |
||||
|
||||
// Otherwise, we have a concrete key trying to override a previous
|
||||
// key, which is *always* wrong.
|
||||
p.panicf("Key '%s' has already been defined.", keyContext) |
||||
} |
||||
hash[key] = value |
||||
} |
||||
|
||||
// setType sets the type of a particular value at a given key.
|
||||
// It should be called immediately AFTER setValue.
|
||||
//
|
||||
// Note that if `key` is empty, then the type given will be applied to the
|
||||
// current context (which is either a table or an array of tables).
|
||||
func (p *parser) setType(key string, typ tomlType) { |
||||
keyContext := make(Key, 0, len(p.context)+1) |
||||
for _, k := range p.context { |
||||
keyContext = append(keyContext, k) |
||||
} |
||||
if len(key) > 0 { // allow type setting for hashes
|
||||
keyContext = append(keyContext, key) |
||||
} |
||||
p.types[keyContext.String()] = typ |
||||
} |
||||
|
||||
// addImplicit sets the given Key as having been created implicitly.
|
||||
func (p *parser) addImplicit(key Key) { |
||||
p.implicits[key.String()] = true |
||||
} |
||||
|
||||
// removeImplicit stops tagging the given key as having been implicitly
|
||||
// created.
|
||||
func (p *parser) removeImplicit(key Key) { |
||||
p.implicits[key.String()] = false |
||||
} |
||||
|
||||
// isImplicit returns true if the key group pointed to by the key was created
|
||||
// implicitly.
|
||||
func (p *parser) isImplicit(key Key) bool { |
||||
return p.implicits[key.String()] |
||||
} |
||||
|
||||
// current returns the full key name of the current context.
|
||||
func (p *parser) current() string { |
||||
if len(p.currentKey) == 0 { |
||||
return p.context.String() |
||||
} |
||||
if len(p.context) == 0 { |
||||
return p.currentKey |
||||
} |
||||
return fmt.Sprintf("%s.%s", p.context, p.currentKey) |
||||
} |
||||
|
||||
func stripFirstNewline(s string) string { |
||||
if len(s) == 0 || s[0] != '\n' { |
||||
return s |
||||
} |
||||
return s[1:] |
||||
} |
||||
|
||||
func stripEscapedWhitespace(s string) string { |
||||
esc := strings.Split(s, "\\\n") |
||||
if len(esc) > 1 { |
||||
for i := 1; i < len(esc); i++ { |
||||
esc[i] = strings.TrimLeftFunc(esc[i], unicode.IsSpace) |
||||
} |
||||
} |
||||
return strings.Join(esc, "") |
||||
} |
||||
|
||||
func (p *parser) replaceEscapes(str string) string { |
||||
var replaced []rune |
||||
s := []byte(str) |
||||
r := 0 |
||||
for r < len(s) { |
||||
if s[r] != '\\' { |
||||
c, size := utf8.DecodeRune(s[r:]) |
||||
r += size |
||||
replaced = append(replaced, c) |
||||
continue |
||||
} |
||||
r += 1 |
||||
if r >= len(s) { |
||||
p.bug("Escape sequence at end of string.") |
||||
return "" |
||||
} |
||||
switch s[r] { |
||||
default: |
||||
p.bug("Expected valid escape code after \\, but got %q.", s[r]) |
||||
return "" |
||||
case 'b': |
||||
replaced = append(replaced, rune(0x0008)) |
||||
r += 1 |
||||
case 't': |
||||
replaced = append(replaced, rune(0x0009)) |
||||
r += 1 |
||||
case 'n': |
||||
replaced = append(replaced, rune(0x000A)) |
||||
r += 1 |
||||
case 'f': |
||||
replaced = append(replaced, rune(0x000C)) |
||||
r += 1 |
||||
case 'r': |
||||
replaced = append(replaced, rune(0x000D)) |
||||
r += 1 |
||||
case '"': |
||||
replaced = append(replaced, rune(0x0022)) |
||||
r += 1 |
||||
case '\\': |
||||
replaced = append(replaced, rune(0x005C)) |
||||
r += 1 |
||||
case 'u': |
||||
// At this point, we know we have a Unicode escape of the form
|
||||
// `uXXXX` at [r, r+5). (Because the lexer guarantees this
|
||||
// for us.)
|
||||
escaped := p.asciiEscapeToUnicode(s[r+1 : r+5]) |
||||
replaced = append(replaced, escaped) |
||||
r += 5 |
||||
case 'U': |
||||
// At this point, we know we have a Unicode escape of the form
|
||||
// `uXXXX` at [r, r+9). (Because the lexer guarantees this
|
||||
// for us.)
|
||||
escaped := p.asciiEscapeToUnicode(s[r+1 : r+9]) |
||||
replaced = append(replaced, escaped) |
||||
r += 9 |
||||
} |
||||
} |
||||
return string(replaced) |
||||
} |
||||
|
||||
func (p *parser) asciiEscapeToUnicode(bs []byte) rune { |
||||
s := string(bs) |
||||
hex, err := strconv.ParseUint(strings.ToLower(s), 16, 32) |
||||
if err != nil { |
||||
p.bug("Could not parse '%s' as a hexadecimal number, but the "+ |
||||
"lexer claims it's OK: %s", s, err) |
||||
} |
||||
if !utf8.ValidRune(rune(hex)) { |
||||
p.panicf("Escaped character '\\u%s' is not valid UTF-8.", s) |
||||
} |
||||
return rune(hex) |
||||
} |
||||
|
||||
func isStringType(ty itemType) bool { |
||||
return ty == itemString || ty == itemMultilineString || |
||||
ty == itemRawString || ty == itemRawMultilineString |
||||
} |
||||
@ -0,0 +1,91 @@ |
||||
package toml |
||||
|
||||
// tomlType represents any Go type that corresponds to a TOML type.
|
||||
// While the first draft of the TOML spec has a simplistic type system that
|
||||
// probably doesn't need this level of sophistication, we seem to be militating
|
||||
// toward adding real composite types.
|
||||
type tomlType interface { |
||||
typeString() string |
||||
} |
||||
|
||||
// typeEqual accepts any two types and returns true if they are equal.
|
||||
func typeEqual(t1, t2 tomlType) bool { |
||||
if t1 == nil || t2 == nil { |
||||
return false |
||||
} |
||||
return t1.typeString() == t2.typeString() |
||||
} |
||||
|
||||
func typeIsHash(t tomlType) bool { |
||||
return typeEqual(t, tomlHash) || typeEqual(t, tomlArrayHash) |
||||
} |
||||
|
||||
type tomlBaseType string |
||||
|
||||
func (btype tomlBaseType) typeString() string { |
||||
return string(btype) |
||||
} |
||||
|
||||
func (btype tomlBaseType) String() string { |
||||
return btype.typeString() |
||||
} |
||||
|
||||
var ( |
||||
tomlInteger tomlBaseType = "Integer" |
||||
tomlFloat tomlBaseType = "Float" |
||||
tomlDatetime tomlBaseType = "Datetime" |
||||
tomlString tomlBaseType = "String" |
||||
tomlBool tomlBaseType = "Bool" |
||||
tomlArray tomlBaseType = "Array" |
||||
tomlHash tomlBaseType = "Hash" |
||||
tomlArrayHash tomlBaseType = "ArrayHash" |
||||
) |
||||
|
||||
// typeOfPrimitive returns a tomlType of any primitive value in TOML.
|
||||
// Primitive values are: Integer, Float, Datetime, String and Bool.
|
||||
//
|
||||
// Passing a lexer item other than the following will cause a BUG message
|
||||
// to occur: itemString, itemBool, itemInteger, itemFloat, itemDatetime.
|
||||
func (p *parser) typeOfPrimitive(lexItem item) tomlType { |
||||
switch lexItem.typ { |
||||
case itemInteger: |
||||
return tomlInteger |
||||
case itemFloat: |
||||
return tomlFloat |
||||
case itemDatetime: |
||||
return tomlDatetime |
||||
case itemString: |
||||
return tomlString |
||||
case itemMultilineString: |
||||
return tomlString |
||||
case itemRawString: |
||||
return tomlString |
||||
case itemRawMultilineString: |
||||
return tomlString |
||||
case itemBool: |
||||
return tomlBool |
||||
} |
||||
p.bug("Cannot infer primitive type of lex item '%s'.", lexItem) |
||||
panic("unreachable") |
||||
} |
||||
|
||||
// typeOfArray returns a tomlType for an array given a list of types of its
|
||||
// values.
|
||||
//
|
||||
// In the current spec, if an array is homogeneous, then its type is always
|
||||
// "Array". If the array is not homogeneous, an error is generated.
|
||||
func (p *parser) typeOfArray(types []tomlType) tomlType { |
||||
// Empty arrays are cool.
|
||||
if len(types) == 0 { |
||||
return tomlArray |
||||
} |
||||
|
||||
theType := types[0] |
||||
for _, t := range types[1:] { |
||||
if !typeEqual(theType, t) { |
||||
p.panicf("Array contains values of type '%s' and '%s', but "+ |
||||
"arrays must be homogeneous.", theType, t) |
||||
} |
||||
} |
||||
return tomlArray |
||||
} |
||||
@ -0,0 +1,242 @@ |
||||
package toml |
||||
|
||||
// Struct field handling is adapted from code in encoding/json:
|
||||
//
|
||||
// Copyright 2010 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the Go distribution.
|
||||
|
||||
import ( |
||||
"reflect" |
||||
"sort" |
||||
"sync" |
||||
) |
||||
|
||||
// A field represents a single field found in a struct.
|
||||
type field struct { |
||||
name string // the name of the field (`toml` tag included)
|
||||
tag bool // whether field has a `toml` tag
|
||||
index []int // represents the depth of an anonymous field
|
||||
typ reflect.Type // the type of the field
|
||||
} |
||||
|
||||
// byName sorts field by name, breaking ties with depth,
|
||||
// then breaking ties with "name came from toml tag", then
|
||||
// breaking ties with index sequence.
|
||||
type byName []field |
||||
|
||||
func (x byName) Len() int { return len(x) } |
||||
|
||||
func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } |
||||
|
||||
func (x byName) Less(i, j int) bool { |
||||
if x[i].name != x[j].name { |
||||
return x[i].name < x[j].name |
||||
} |
||||
if len(x[i].index) != len(x[j].index) { |
||||
return len(x[i].index) < len(x[j].index) |
||||
} |
||||
if x[i].tag != x[j].tag { |
||||
return x[i].tag |
||||
} |
||||
return byIndex(x).Less(i, j) |
||||
} |
||||
|
||||
// byIndex sorts field by index sequence.
|
||||
type byIndex []field |
||||
|
||||
func (x byIndex) Len() int { return len(x) } |
||||
|
||||
func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] } |
||||
|
||||
func (x byIndex) Less(i, j int) bool { |
||||
for k, xik := range x[i].index { |
||||
if k >= len(x[j].index) { |
||||
return false |
||||
} |
||||
if xik != x[j].index[k] { |
||||
return xik < x[j].index[k] |
||||
} |
||||
} |
||||
return len(x[i].index) < len(x[j].index) |
||||
} |
||||
|
||||
// typeFields returns a list of fields that TOML should recognize for the given
|
||||
// type. The algorithm is breadth-first search over the set of structs to
|
||||
// include - the top struct and then any reachable anonymous structs.
|
||||
func typeFields(t reflect.Type) []field { |
||||
// Anonymous fields to explore at the current level and the next.
|
||||
current := []field{} |
||||
next := []field{{typ: t}} |
||||
|
||||
// Count of queued names for current level and the next.
|
||||
count := map[reflect.Type]int{} |
||||
nextCount := map[reflect.Type]int{} |
||||
|
||||
// Types already visited at an earlier level.
|
||||
visited := map[reflect.Type]bool{} |
||||
|
||||
// Fields found.
|
||||
var fields []field |
||||
|
||||
for len(next) > 0 { |
||||
current, next = next, current[:0] |
||||
count, nextCount = nextCount, map[reflect.Type]int{} |
||||
|
||||
for _, f := range current { |
||||
if visited[f.typ] { |
||||
continue |
||||
} |
||||
visited[f.typ] = true |
||||
|
||||
// Scan f.typ for fields to include.
|
||||
for i := 0; i < f.typ.NumField(); i++ { |
||||
sf := f.typ.Field(i) |
||||
if sf.PkgPath != "" && !sf.Anonymous { // unexported
|
||||
continue |
||||
} |
||||
opts := getOptions(sf.Tag) |
||||
if opts.skip { |
||||
continue |
||||
} |
||||
index := make([]int, len(f.index)+1) |
||||
copy(index, f.index) |
||||
index[len(f.index)] = i |
||||
|
||||
ft := sf.Type |
||||
if ft.Name() == "" && ft.Kind() == reflect.Ptr { |
||||
// Follow pointer.
|
||||
ft = ft.Elem() |
||||
} |
||||
|
||||
// Record found field and index sequence.
|
||||
if opts.name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { |
||||
tagged := opts.name != "" |
||||
name := opts.name |
||||
if name == "" { |
||||
name = sf.Name |
||||
} |
||||
fields = append(fields, field{name, tagged, index, ft}) |
||||
if count[f.typ] > 1 { |
||||
// If there were multiple instances, add a second,
|
||||
// so that the annihilation code will see a duplicate.
|
||||
// It only cares about the distinction between 1 or 2,
|
||||
// so don't bother generating any more copies.
|
||||
fields = append(fields, fields[len(fields)-1]) |
||||
} |
||||
continue |
||||
} |
||||
|
||||
// Record new anonymous struct to explore in next round.
|
||||
nextCount[ft]++ |
||||
if nextCount[ft] == 1 { |
||||
f := field{name: ft.Name(), index: index, typ: ft} |
||||
next = append(next, f) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
sort.Sort(byName(fields)) |
||||
|
||||
// Delete all fields that are hidden by the Go rules for embedded fields,
|
||||
// except that fields with TOML tags are promoted.
|
||||
|
||||
// The fields are sorted in primary order of name, secondary order
|
||||
// of field index length. Loop over names; for each name, delete
|
||||
// hidden fields by choosing the one dominant field that survives.
|
||||
out := fields[:0] |
||||
for advance, i := 0, 0; i < len(fields); i += advance { |
||||
// One iteration per name.
|
||||
// Find the sequence of fields with the name of this first field.
|
||||
fi := fields[i] |
||||
name := fi.name |
||||
for advance = 1; i+advance < len(fields); advance++ { |
||||
fj := fields[i+advance] |
||||
if fj.name != name { |
||||
break |
||||
} |
||||
} |
||||
if advance == 1 { // Only one field with this name
|
||||
out = append(out, fi) |
||||
continue |
||||
} |
||||
dominant, ok := dominantField(fields[i : i+advance]) |
||||
if ok { |
||||
out = append(out, dominant) |
||||
} |
||||
} |
||||
|
||||
fields = out |
||||
sort.Sort(byIndex(fields)) |
||||
|
||||
return fields |
||||
} |
||||
|
||||
// dominantField looks through the fields, all of which are known to
|
||||
// have the same name, to find the single field that dominates the
|
||||
// others using Go's embedding rules, modified by the presence of
|
||||
// TOML tags. If there are multiple top-level fields, the boolean
|
||||
// will be false: This condition is an error in Go and we skip all
|
||||
// the fields.
|
||||
func dominantField(fields []field) (field, bool) { |
||||
// The fields are sorted in increasing index-length order. The winner
|
||||
// must therefore be one with the shortest index length. Drop all
|
||||
// longer entries, which is easy: just truncate the slice.
|
||||
length := len(fields[0].index) |
||||
tagged := -1 // Index of first tagged field.
|
||||
for i, f := range fields { |
||||
if len(f.index) > length { |
||||
fields = fields[:i] |
||||
break |
||||
} |
||||
if f.tag { |
||||
if tagged >= 0 { |
||||
// Multiple tagged fields at the same level: conflict.
|
||||
// Return no field.
|
||||
return field{}, false |
||||
} |
||||
tagged = i |
||||
} |
||||
} |
||||
if tagged >= 0 { |
||||
return fields[tagged], true |
||||
} |
||||
// All remaining fields have the same length. If there's more than one,
|
||||
// we have a conflict (two fields named "X" at the same level) and we
|
||||
// return no field.
|
||||
if len(fields) > 1 { |
||||
return field{}, false |
||||
} |
||||
return fields[0], true |
||||
} |
||||
|
||||
var fieldCache struct { |
||||
sync.RWMutex |
||||
m map[reflect.Type][]field |
||||
} |
||||
|
||||
// cachedTypeFields is like typeFields but uses a cache to avoid repeated work.
|
||||
func cachedTypeFields(t reflect.Type) []field { |
||||
fieldCache.RLock() |
||||
f := fieldCache.m[t] |
||||
fieldCache.RUnlock() |
||||
if f != nil { |
||||
return f |
||||
} |
||||
|
||||
// Compute fields without lock.
|
||||
// Might duplicate effort but won't hold other computations back.
|
||||
f = typeFields(t) |
||||
if f == nil { |
||||
f = []field{} |
||||
} |
||||
|
||||
fieldCache.Lock() |
||||
if fieldCache.m == nil { |
||||
fieldCache.m = map[reflect.Type][]field{} |
||||
} |
||||
fieldCache.m[t] = f |
||||
fieldCache.Unlock() |
||||
return f |
||||
} |
||||
@ -0,0 +1,19 @@ |
||||
Copyright (c) 2013 Dustin Sallings |
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||
of this software and associated documentation files (the "Software"), to deal |
||||
in the Software without restriction, including without limitation the rights |
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||
copies of the Software, and to permit persons to whom the Software is |
||||
furnished to do so, subject to the following conditions: |
||||
|
||||
The above copyright notice and this permission notice shall be included in |
||||
all copies or substantial portions of the Software. |
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
||||
THE SOFTWARE. |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,333 @@ |
||||
package memcached |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/binary" |
||||
"fmt" |
||||
"io" |
||||
"math" |
||||
|
||||
"github.com/couchbase/gomemcached" |
||||
"github.com/couchbase/goutils/logging" |
||||
) |
||||
|
||||
// TAP protocol docs: <http://www.couchbase.com/wiki/display/couchbase/TAP+Protocol>
|
||||
|
||||
// TapOpcode is the tap operation type (found in TapEvent)
|
||||
type TapOpcode uint8 |
||||
|
||||
// Tap opcode values.
|
||||
const ( |
||||
TapBeginBackfill = TapOpcode(iota) |
||||
TapEndBackfill |
||||
TapMutation |
||||
TapDeletion |
||||
TapCheckpointStart |
||||
TapCheckpointEnd |
||||
tapEndStream |
||||
) |
||||
|
||||
const tapMutationExtraLen = 16 |
||||
|
||||
var tapOpcodeNames map[TapOpcode]string |
||||
|
||||
func init() { |
||||
tapOpcodeNames = map[TapOpcode]string{ |
||||
TapBeginBackfill: "BeginBackfill", |
||||
TapEndBackfill: "EndBackfill", |
||||
TapMutation: "Mutation", |
||||
TapDeletion: "Deletion", |
||||
TapCheckpointStart: "TapCheckpointStart", |
||||
TapCheckpointEnd: "TapCheckpointEnd", |
||||
tapEndStream: "EndStream", |
||||
} |
||||
} |
||||
|
||||
func (opcode TapOpcode) String() string { |
||||
name := tapOpcodeNames[opcode] |
||||
if name == "" { |
||||
name = fmt.Sprintf("#%d", opcode) |
||||
} |
||||
return name |
||||
} |
||||
|
||||
// TapEvent is a TAP notification of an operation on the server.
|
||||
type TapEvent struct { |
||||
Opcode TapOpcode // Type of event
|
||||
VBucket uint16 // VBucket this event applies to
|
||||
Flags uint32 // Item flags
|
||||
Expiry uint32 // Item expiration time
|
||||
Key, Value []byte // Item key/value
|
||||
Cas uint64 |
||||
} |
||||
|
||||
func makeTapEvent(req gomemcached.MCRequest) *TapEvent { |
||||
event := TapEvent{ |
||||
VBucket: req.VBucket, |
||||
} |
||||
switch req.Opcode { |
||||
case gomemcached.TAP_MUTATION: |
||||
event.Opcode = TapMutation |
||||
event.Key = req.Key |
||||
event.Value = req.Body |
||||
event.Cas = req.Cas |
||||
case gomemcached.TAP_DELETE: |
||||
event.Opcode = TapDeletion |
||||
event.Key = req.Key |
||||
event.Cas = req.Cas |
||||
case gomemcached.TAP_CHECKPOINT_START: |
||||
event.Opcode = TapCheckpointStart |
||||
case gomemcached.TAP_CHECKPOINT_END: |
||||
event.Opcode = TapCheckpointEnd |
||||
case gomemcached.TAP_OPAQUE: |
||||
if len(req.Extras) < 8+4 { |
||||
return nil |
||||
} |
||||
switch op := int(binary.BigEndian.Uint32(req.Extras[8:])); op { |
||||
case gomemcached.TAP_OPAQUE_INITIAL_VBUCKET_STREAM: |
||||
event.Opcode = TapBeginBackfill |
||||
case gomemcached.TAP_OPAQUE_CLOSE_BACKFILL: |
||||
event.Opcode = TapEndBackfill |
||||
case gomemcached.TAP_OPAQUE_CLOSE_TAP_STREAM: |
||||
event.Opcode = tapEndStream |
||||
case gomemcached.TAP_OPAQUE_ENABLE_AUTO_NACK: |
||||
return nil |
||||
case gomemcached.TAP_OPAQUE_ENABLE_CHECKPOINT_SYNC: |
||||
return nil |
||||
default: |
||||
logging.Infof("TapFeed: Ignoring TAP_OPAQUE/%d", op) |
||||
return nil // unknown opaque event
|
||||
} |
||||
case gomemcached.NOOP: |
||||
return nil // ignore
|
||||
default: |
||||
logging.Infof("TapFeed: Ignoring %s", req.Opcode) |
||||
return nil // unknown event
|
||||
} |
||||
|
||||
if len(req.Extras) >= tapMutationExtraLen && |
||||
(event.Opcode == TapMutation || event.Opcode == TapDeletion) { |
||||
|
||||
event.Flags = binary.BigEndian.Uint32(req.Extras[8:]) |
||||
event.Expiry = binary.BigEndian.Uint32(req.Extras[12:]) |
||||
} |
||||
|
||||
return &event |
||||
} |
||||
|
||||
func (event TapEvent) String() string { |
||||
switch event.Opcode { |
||||
case TapBeginBackfill, TapEndBackfill, TapCheckpointStart, TapCheckpointEnd: |
||||
return fmt.Sprintf("<TapEvent %s, vbucket=%d>", |
||||
event.Opcode, event.VBucket) |
||||
default: |
||||
return fmt.Sprintf("<TapEvent %s, key=%q (%d bytes) flags=%x, exp=%d>", |
||||
event.Opcode, event.Key, len(event.Value), |
||||
event.Flags, event.Expiry) |
||||
} |
||||
} |
||||
|
||||
// TapArguments are parameters for requesting a TAP feed.
|
||||
//
|
||||
// Call DefaultTapArguments to get a default one.
|
||||
type TapArguments struct { |
||||
// Timestamp of oldest item to send.
|
||||
//
|
||||
// Use TapNoBackfill to suppress all past items.
|
||||
Backfill uint64 |
||||
// If set, server will disconnect after sending existing items.
|
||||
Dump bool |
||||
// The indices of the vbuckets to watch; empty/nil to watch all.
|
||||
VBuckets []uint16 |
||||
// Transfers ownership of vbuckets during cluster rebalance.
|
||||
Takeover bool |
||||
// If true, server will wait for client ACK after every notification.
|
||||
SupportAck bool |
||||
// If true, client doesn't want values so server shouldn't send them.
|
||||
KeysOnly bool |
||||
// If true, client wants the server to send checkpoint events.
|
||||
Checkpoint bool |
||||
// Optional identifier to use for this client, to allow reconnects
|
||||
ClientName string |
||||
// Registers this client (by name) till explicitly deregistered.
|
||||
RegisteredClient bool |
||||
} |
||||
|
||||
// Value for TapArguments.Backfill denoting that no past events at all
|
||||
// should be sent.
|
||||
const TapNoBackfill = math.MaxUint64 |
||||
|
||||
// DefaultTapArguments returns a default set of parameter values to
|
||||
// pass to StartTapFeed.
|
||||
func DefaultTapArguments() TapArguments { |
||||
return TapArguments{ |
||||
Backfill: TapNoBackfill, |
||||
} |
||||
} |
||||
|
||||
func (args *TapArguments) flags() []byte { |
||||
var flags gomemcached.TapConnectFlag |
||||
if args.Backfill != 0 { |
||||
flags |= gomemcached.BACKFILL |
||||
} |
||||
if args.Dump { |
||||
flags |= gomemcached.DUMP |
||||
} |
||||
if len(args.VBuckets) > 0 { |
||||
flags |= gomemcached.LIST_VBUCKETS |
||||
} |
||||
if args.Takeover { |
||||
flags |= gomemcached.TAKEOVER_VBUCKETS |
||||
} |
||||
if args.SupportAck { |
||||
flags |= gomemcached.SUPPORT_ACK |
||||
} |
||||
if args.KeysOnly { |
||||
flags |= gomemcached.REQUEST_KEYS_ONLY |
||||
} |
||||
if args.Checkpoint { |
||||
flags |= gomemcached.CHECKPOINT |
||||
} |
||||
if args.RegisteredClient { |
||||
flags |= gomemcached.REGISTERED_CLIENT |
||||
} |
||||
encoded := make([]byte, 4) |
||||
binary.BigEndian.PutUint32(encoded, uint32(flags)) |
||||
return encoded |
||||
} |
||||
|
||||
func must(err error) { |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
} |
||||
|
||||
func (args *TapArguments) bytes() (rv []byte) { |
||||
buf := bytes.NewBuffer([]byte{}) |
||||
|
||||
if args.Backfill > 0 { |
||||
must(binary.Write(buf, binary.BigEndian, uint64(args.Backfill))) |
||||
} |
||||
|
||||
if len(args.VBuckets) > 0 { |
||||
must(binary.Write(buf, binary.BigEndian, uint16(len(args.VBuckets)))) |
||||
for i := 0; i < len(args.VBuckets); i++ { |
||||
must(binary.Write(buf, binary.BigEndian, uint16(args.VBuckets[i]))) |
||||
} |
||||
} |
||||
return buf.Bytes() |
||||
} |
||||
|
||||
// TapFeed represents a stream of events from a server.
|
||||
type TapFeed struct { |
||||
C <-chan TapEvent |
||||
Error error |
||||
closer chan bool |
||||
} |
||||
|
||||
// StartTapFeed starts a TAP feed on a client connection.
|
||||
//
|
||||
// The events can be read from the returned channel. The connection
|
||||
// can no longer be used for other purposes; it's now reserved for
|
||||
// receiving the TAP messages. To stop receiving events, close the
|
||||
// client connection.
|
||||
func (mc *Client) StartTapFeed(args TapArguments) (*TapFeed, error) { |
||||
rq := &gomemcached.MCRequest{ |
||||
Opcode: gomemcached.TAP_CONNECT, |
||||
Key: []byte(args.ClientName), |
||||
Extras: args.flags(), |
||||
Body: args.bytes()} |
||||
|
||||
err := mc.Transmit(rq) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
ch := make(chan TapEvent) |
||||
feed := &TapFeed{ |
||||
C: ch, |
||||
closer: make(chan bool), |
||||
} |
||||
go mc.runFeed(ch, feed) |
||||
return feed, nil |
||||
} |
||||
|
||||
// TapRecvHook is called after every incoming tap packet is received.
|
||||
var TapRecvHook func(*gomemcached.MCRequest, int, error) |
||||
|
||||
// Internal goroutine that reads from the socket and writes events to
|
||||
// the channel
|
||||
func (mc *Client) runFeed(ch chan TapEvent, feed *TapFeed) { |
||||
defer close(ch) |
||||
var headerBuf [gomemcached.HDR_LEN]byte |
||||
loop: |
||||
for { |
||||
// Read the next request from the server.
|
||||
//
|
||||
// (Can't call mc.Receive() because it reads a
|
||||
// _response_ not a request.)
|
||||
var pkt gomemcached.MCRequest |
||||
n, err := pkt.Receive(mc.conn, headerBuf[:]) |
||||
if TapRecvHook != nil { |
||||
TapRecvHook(&pkt, n, err) |
||||
} |
||||
|
||||
if err != nil { |
||||
if err != io.EOF { |
||||
feed.Error = err |
||||
} |
||||
break loop |
||||
} |
||||
|
||||
//logging.Infof("** TapFeed received %#v : %q", pkt, pkt.Body)
|
||||
|
||||
if pkt.Opcode == gomemcached.TAP_CONNECT { |
||||
// This is not an event from the server; it's
|
||||
// an error response to my connect request.
|
||||
feed.Error = fmt.Errorf("tap connection failed: %s", pkt.Body) |
||||
break loop |
||||
} |
||||
|
||||
event := makeTapEvent(pkt) |
||||
if event != nil { |
||||
if event.Opcode == tapEndStream { |
||||
break loop |
||||
} |
||||
|
||||
select { |
||||
case ch <- *event: |
||||
case <-feed.closer: |
||||
break loop |
||||
} |
||||
} |
||||
|
||||
if len(pkt.Extras) >= 4 { |
||||
reqFlags := binary.BigEndian.Uint16(pkt.Extras[2:]) |
||||
if reqFlags&gomemcached.TAP_ACK != 0 { |
||||
if _, err := mc.sendAck(&pkt); err != nil { |
||||
feed.Error = err |
||||
break loop |
||||
} |
||||
} |
||||
} |
||||
} |
||||
if err := mc.Close(); err != nil { |
||||
logging.Errorf("Error closing memcached client: %v", err) |
||||
} |
||||
} |
||||
|
||||
func (mc *Client) sendAck(pkt *gomemcached.MCRequest) (int, error) { |
||||
res := gomemcached.MCResponse{ |
||||
Opcode: pkt.Opcode, |
||||
Opaque: pkt.Opaque, |
||||
Status: gomemcached.SUCCESS, |
||||
} |
||||
return res.Transmit(mc.conn) |
||||
} |
||||
|
||||
// Close terminates a TapFeed.
|
||||
//
|
||||
// Call this if you stop using a TapFeed before its channel ends.
|
||||
func (feed *TapFeed) Close() { |
||||
close(feed.closer) |
||||
} |
||||
@ -0,0 +1,67 @@ |
||||
package memcached |
||||
|
||||
import ( |
||||
"errors" |
||||
"io" |
||||
|
||||
"github.com/couchbase/gomemcached" |
||||
) |
||||
|
||||
var errNoConn = errors.New("no connection") |
||||
|
||||
// UnwrapMemcachedError converts memcached errors to normal responses.
|
||||
//
|
||||
// If the error is a memcached response, declare the error to be nil
|
||||
// so a client can handle the status without worrying about whether it
|
||||
// indicates success or failure.
|
||||
func UnwrapMemcachedError(rv *gomemcached.MCResponse, |
||||
err error) (*gomemcached.MCResponse, error) { |
||||
|
||||
if rv == err { |
||||
return rv, nil |
||||
} |
||||
return rv, err |
||||
} |
||||
|
||||
// ReceiveHook is called after every packet is received (or attempted to be)
|
||||
var ReceiveHook func(*gomemcached.MCResponse, int, error) |
||||
|
||||
func getResponse(s io.Reader, hdrBytes []byte) (rv *gomemcached.MCResponse, n int, err error) { |
||||
if s == nil { |
||||
return nil, 0, errNoConn |
||||
} |
||||
|
||||
rv = &gomemcached.MCResponse{} |
||||
n, err = rv.Receive(s, hdrBytes) |
||||
|
||||
if ReceiveHook != nil { |
||||
ReceiveHook(rv, n, err) |
||||
} |
||||
|
||||
if err == nil && (rv.Status != gomemcached.SUCCESS && rv.Status != gomemcached.AUTH_CONTINUE) { |
||||
err = rv |
||||
} |
||||
return rv, n, err |
||||
} |
||||
|
||||
// TransmitHook is called after each packet is transmitted.
|
||||
var TransmitHook func(*gomemcached.MCRequest, int, error) |
||||
|
||||
func transmitRequest(o io.Writer, req *gomemcached.MCRequest) (int, error) { |
||||
if o == nil { |
||||
return 0, errNoConn |
||||
} |
||||
n, err := req.Transmit(o) |
||||
if TransmitHook != nil { |
||||
TransmitHook(req, n, err) |
||||
} |
||||
return n, err |
||||
} |
||||
|
||||
func transmitResponse(o io.Writer, res *gomemcached.MCResponse) (int, error) { |
||||
if o == nil { |
||||
return 0, errNoConn |
||||
} |
||||
n, err := res.Transmit(o) |
||||
return n, err |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,335 @@ |
||||
// Package gomemcached is binary protocol packet formats and constants.
|
||||
package gomemcached |
||||
|
||||
import ( |
||||
"fmt" |
||||
) |
||||
|
||||
const ( |
||||
REQ_MAGIC = 0x80 |
||||
RES_MAGIC = 0x81 |
||||
) |
||||
|
||||
// CommandCode for memcached packets.
|
||||
type CommandCode uint8 |
||||
|
||||
const ( |
||||
GET = CommandCode(0x00) |
||||
SET = CommandCode(0x01) |
||||
ADD = CommandCode(0x02) |
||||
REPLACE = CommandCode(0x03) |
||||
DELETE = CommandCode(0x04) |
||||
INCREMENT = CommandCode(0x05) |
||||
DECREMENT = CommandCode(0x06) |
||||
QUIT = CommandCode(0x07) |
||||
FLUSH = CommandCode(0x08) |
||||
GETQ = CommandCode(0x09) |
||||
NOOP = CommandCode(0x0a) |
||||
VERSION = CommandCode(0x0b) |
||||
GETK = CommandCode(0x0c) |
||||
GETKQ = CommandCode(0x0d) |
||||
APPEND = CommandCode(0x0e) |
||||
PREPEND = CommandCode(0x0f) |
||||
STAT = CommandCode(0x10) |
||||
SETQ = CommandCode(0x11) |
||||
ADDQ = CommandCode(0x12) |
||||
REPLACEQ = CommandCode(0x13) |
||||
DELETEQ = CommandCode(0x14) |
||||
INCREMENTQ = CommandCode(0x15) |
||||
DECREMENTQ = CommandCode(0x16) |
||||
QUITQ = CommandCode(0x17) |
||||
FLUSHQ = CommandCode(0x18) |
||||
APPENDQ = CommandCode(0x19) |
||||
AUDIT = CommandCode(0x27) |
||||
PREPENDQ = CommandCode(0x1a) |
||||
GAT = CommandCode(0x1d) |
||||
HELLO = CommandCode(0x1f) |
||||
RGET = CommandCode(0x30) |
||||
RSET = CommandCode(0x31) |
||||
RSETQ = CommandCode(0x32) |
||||
RAPPEND = CommandCode(0x33) |
||||
RAPPENDQ = CommandCode(0x34) |
||||
RPREPEND = CommandCode(0x35) |
||||
RPREPENDQ = CommandCode(0x36) |
||||
RDELETE = CommandCode(0x37) |
||||
RDELETEQ = CommandCode(0x38) |
||||
RINCR = CommandCode(0x39) |
||||
RINCRQ = CommandCode(0x3a) |
||||
RDECR = CommandCode(0x3b) |
||||
RDECRQ = CommandCode(0x3c) |
||||
|
||||
SASL_LIST_MECHS = CommandCode(0x20) |
||||
SASL_AUTH = CommandCode(0x21) |
||||
SASL_STEP = CommandCode(0x22) |
||||
|
||||
SET_VBUCKET = CommandCode(0x3d) |
||||
|
||||
TAP_CONNECT = CommandCode(0x40) // Client-sent request to initiate Tap feed
|
||||
TAP_MUTATION = CommandCode(0x41) // Notification of a SET/ADD/REPLACE/etc. on the server
|
||||
TAP_DELETE = CommandCode(0x42) // Notification of a DELETE on the server
|
||||
TAP_FLUSH = CommandCode(0x43) // Replicates a flush_all command
|
||||
TAP_OPAQUE = CommandCode(0x44) // Opaque control data from the engine
|
||||
TAP_VBUCKET_SET = CommandCode(0x45) // Sets state of vbucket in receiver (used in takeover)
|
||||
TAP_CHECKPOINT_START = CommandCode(0x46) // Notifies start of new checkpoint
|
||||
TAP_CHECKPOINT_END = CommandCode(0x47) // Notifies end of checkpoint
|
||||
|
||||
UPR_OPEN = CommandCode(0x50) // Open a UPR connection with a name
|
||||
UPR_ADDSTREAM = CommandCode(0x51) // Sent by ebucketMigrator to UPR Consumer
|
||||
UPR_CLOSESTREAM = CommandCode(0x52) // Sent by eBucketMigrator to UPR Consumer
|
||||
UPR_FAILOVERLOG = CommandCode(0x54) // Request failover logs
|
||||
UPR_STREAMREQ = CommandCode(0x53) // Stream request from consumer to producer
|
||||
UPR_STREAMEND = CommandCode(0x55) // Sent by producer when it has no more messages to stream
|
||||
UPR_SNAPSHOT = CommandCode(0x56) // Start of a new snapshot
|
||||
UPR_MUTATION = CommandCode(0x57) // Key mutation
|
||||
UPR_DELETION = CommandCode(0x58) // Key deletion
|
||||
UPR_EXPIRATION = CommandCode(0x59) // Key expiration
|
||||
UPR_FLUSH = CommandCode(0x5a) // Delete all the data for a vbucket
|
||||
UPR_NOOP = CommandCode(0x5c) // UPR NOOP
|
||||
UPR_BUFFERACK = CommandCode(0x5d) // UPR Buffer Acknowledgement
|
||||
UPR_CONTROL = CommandCode(0x5e) // Set flow control params
|
||||
|
||||
SELECT_BUCKET = CommandCode(0x89) // Select bucket
|
||||
|
||||
OBSERVE_SEQNO = CommandCode(0x91) // Sequence Number based Observe
|
||||
OBSERVE = CommandCode(0x92) |
||||
|
||||
GET_META = CommandCode(0xA0) // Get meta. returns with expiry, flags, cas etc
|
||||
SUBDOC_GET = CommandCode(0xc5) // Get subdoc. Returns with xattrs
|
||||
SUBDOC_MULTI_LOOKUP = CommandCode(0xd0) // Multi lookup. Doc xattrs and meta.
|
||||
) |
||||
|
||||
// command codes that are counted toward DCP control buffer
|
||||
// when DCP clients receive DCP messages with these command codes, they need to provide acknowledgement
|
||||
var BufferedCommandCodeMap = map[CommandCode]bool{ |
||||
SET_VBUCKET: true, |
||||
UPR_STREAMEND: true, |
||||
UPR_SNAPSHOT: true, |
||||
UPR_MUTATION: true, |
||||
UPR_DELETION: true, |
||||
UPR_EXPIRATION: true} |
||||
|
||||
// Status field for memcached response.
|
||||
type Status uint16 |
||||
|
||||
// Matches with protocol_binary.h as source of truth
|
||||
const ( |
||||
SUCCESS = Status(0x00) |
||||
KEY_ENOENT = Status(0x01) |
||||
KEY_EEXISTS = Status(0x02) |
||||
E2BIG = Status(0x03) |
||||
EINVAL = Status(0x04) |
||||
NOT_STORED = Status(0x05) |
||||
DELTA_BADVAL = Status(0x06) |
||||
NOT_MY_VBUCKET = Status(0x07) |
||||
NO_BUCKET = Status(0x08) |
||||
LOCKED = Status(0x09) |
||||
AUTH_STALE = Status(0x1f) |
||||
AUTH_ERROR = Status(0x20) |
||||
AUTH_CONTINUE = Status(0x21) |
||||
ERANGE = Status(0x22) |
||||
ROLLBACK = Status(0x23) |
||||
EACCESS = Status(0x24) |
||||
NOT_INITIALIZED = Status(0x25) |
||||
UNKNOWN_COMMAND = Status(0x81) |
||||
ENOMEM = Status(0x82) |
||||
NOT_SUPPORTED = Status(0x83) |
||||
EINTERNAL = Status(0x84) |
||||
EBUSY = Status(0x85) |
||||
TMPFAIL = Status(0x86) |
||||
|
||||
// SUBDOC
|
||||
SUBDOC_PATH_NOT_FOUND = Status(0xc0) |
||||
SUBDOC_BAD_MULTI = Status(0xcc) |
||||
SUBDOC_MULTI_PATH_FAILURE_DELETED = Status(0xd3) |
||||
) |
||||
|
||||
// for log redaction
|
||||
const ( |
||||
UdTagBegin = "<ud>" |
||||
UdTagEnd = "</ud>" |
||||
) |
||||
|
||||
var isFatal = map[Status]bool{ |
||||
DELTA_BADVAL: true, |
||||
NO_BUCKET: true, |
||||
AUTH_STALE: true, |
||||
AUTH_ERROR: true, |
||||
ERANGE: true, |
||||
ROLLBACK: true, |
||||
EACCESS: true, |
||||
ENOMEM: true, |
||||
NOT_SUPPORTED: true, |
||||
} |
||||
|
||||
// the producer/consumer bit in dcp flags
|
||||
var DCP_PRODUCER uint32 = 0x01 |
||||
|
||||
// the include XATTRS bit in dcp flags
|
||||
var DCP_OPEN_INCLUDE_XATTRS uint32 = 0x04 |
||||
|
||||
// the include deletion time bit in dcp flags
|
||||
var DCP_OPEN_INCLUDE_DELETE_TIMES uint32 = 0x20 |
||||
|
||||
// Datatype to Include XATTRS in SUBDOC GET
|
||||
var SUBDOC_FLAG_XATTR uint8 = 0x04 |
||||
|
||||
// MCItem is an internal representation of an item.
|
||||
type MCItem struct { |
||||
Cas uint64 |
||||
Flags, Expiration uint32 |
||||
Data []byte |
||||
} |
||||
|
||||
// Number of bytes in a binary protocol header.
|
||||
const HDR_LEN = 24 |
||||
|
||||
// Mapping of CommandCode -> name of command (not exhaustive)
|
||||
var CommandNames map[CommandCode]string |
||||
|
||||
// StatusNames human readable names for memcached response.
|
||||
var StatusNames map[Status]string |
||||
|
||||
func init() { |
||||
CommandNames = make(map[CommandCode]string) |
||||
CommandNames[GET] = "GET" |
||||
CommandNames[SET] = "SET" |
||||
CommandNames[ADD] = "ADD" |
||||
CommandNames[REPLACE] = "REPLACE" |
||||
CommandNames[DELETE] = "DELETE" |
||||
CommandNames[INCREMENT] = "INCREMENT" |
||||
CommandNames[DECREMENT] = "DECREMENT" |
||||
CommandNames[QUIT] = "QUIT" |
||||
CommandNames[FLUSH] = "FLUSH" |
||||
CommandNames[GETQ] = "GETQ" |
||||
CommandNames[NOOP] = "NOOP" |
||||
CommandNames[VERSION] = "VERSION" |
||||
CommandNames[GETK] = "GETK" |
||||
CommandNames[GETKQ] = "GETKQ" |
||||
CommandNames[APPEND] = "APPEND" |
||||
CommandNames[PREPEND] = "PREPEND" |
||||
CommandNames[STAT] = "STAT" |
||||
CommandNames[SETQ] = "SETQ" |
||||
CommandNames[ADDQ] = "ADDQ" |
||||
CommandNames[REPLACEQ] = "REPLACEQ" |
||||
CommandNames[DELETEQ] = "DELETEQ" |
||||
CommandNames[INCREMENTQ] = "INCREMENTQ" |
||||
CommandNames[DECREMENTQ] = "DECREMENTQ" |
||||
CommandNames[QUITQ] = "QUITQ" |
||||
CommandNames[FLUSHQ] = "FLUSHQ" |
||||
CommandNames[APPENDQ] = "APPENDQ" |
||||
CommandNames[PREPENDQ] = "PREPENDQ" |
||||
CommandNames[RGET] = "RGET" |
||||
CommandNames[RSET] = "RSET" |
||||
CommandNames[RSETQ] = "RSETQ" |
||||
CommandNames[RAPPEND] = "RAPPEND" |
||||
CommandNames[RAPPENDQ] = "RAPPENDQ" |
||||
CommandNames[RPREPEND] = "RPREPEND" |
||||
CommandNames[RPREPENDQ] = "RPREPENDQ" |
||||
CommandNames[RDELETE] = "RDELETE" |
||||
CommandNames[RDELETEQ] = "RDELETEQ" |
||||
CommandNames[RINCR] = "RINCR" |
||||
CommandNames[RINCRQ] = "RINCRQ" |
||||
CommandNames[RDECR] = "RDECR" |
||||
CommandNames[RDECRQ] = "RDECRQ" |
||||
|
||||
CommandNames[SASL_LIST_MECHS] = "SASL_LIST_MECHS" |
||||
CommandNames[SASL_AUTH] = "SASL_AUTH" |
||||
CommandNames[SASL_STEP] = "SASL_STEP" |
||||
|
||||
CommandNames[TAP_CONNECT] = "TAP_CONNECT" |
||||
CommandNames[TAP_MUTATION] = "TAP_MUTATION" |
||||
CommandNames[TAP_DELETE] = "TAP_DELETE" |
||||
CommandNames[TAP_FLUSH] = "TAP_FLUSH" |
||||
CommandNames[TAP_OPAQUE] = "TAP_OPAQUE" |
||||
CommandNames[TAP_VBUCKET_SET] = "TAP_VBUCKET_SET" |
||||
CommandNames[TAP_CHECKPOINT_START] = "TAP_CHECKPOINT_START" |
||||
CommandNames[TAP_CHECKPOINT_END] = "TAP_CHECKPOINT_END" |
||||
|
||||
CommandNames[UPR_OPEN] = "UPR_OPEN" |
||||
CommandNames[UPR_ADDSTREAM] = "UPR_ADDSTREAM" |
||||
CommandNames[UPR_CLOSESTREAM] = "UPR_CLOSESTREAM" |
||||
CommandNames[UPR_FAILOVERLOG] = "UPR_FAILOVERLOG" |
||||
CommandNames[UPR_STREAMREQ] = "UPR_STREAMREQ" |
||||
CommandNames[UPR_STREAMEND] = "UPR_STREAMEND" |
||||
CommandNames[UPR_SNAPSHOT] = "UPR_SNAPSHOT" |
||||
CommandNames[UPR_MUTATION] = "UPR_MUTATION" |
||||
CommandNames[UPR_DELETION] = "UPR_DELETION" |
||||
CommandNames[UPR_EXPIRATION] = "UPR_EXPIRATION" |
||||
CommandNames[UPR_FLUSH] = "UPR_FLUSH" |
||||
CommandNames[UPR_NOOP] = "UPR_NOOP" |
||||
CommandNames[UPR_BUFFERACK] = "UPR_BUFFERACK" |
||||
CommandNames[UPR_CONTROL] = "UPR_CONTROL" |
||||
CommandNames[SUBDOC_GET] = "SUBDOC_GET" |
||||
CommandNames[SUBDOC_MULTI_LOOKUP] = "SUBDOC_MULTI_LOOKUP" |
||||
|
||||
StatusNames = make(map[Status]string) |
||||
StatusNames[SUCCESS] = "SUCCESS" |
||||
StatusNames[KEY_ENOENT] = "KEY_ENOENT" |
||||
StatusNames[KEY_EEXISTS] = "KEY_EEXISTS" |
||||
StatusNames[E2BIG] = "E2BIG" |
||||
StatusNames[EINVAL] = "EINVAL" |
||||
StatusNames[NOT_STORED] = "NOT_STORED" |
||||
StatusNames[DELTA_BADVAL] = "DELTA_BADVAL" |
||||
StatusNames[NOT_MY_VBUCKET] = "NOT_MY_VBUCKET" |
||||
StatusNames[NO_BUCKET] = "NO_BUCKET" |
||||
StatusNames[AUTH_STALE] = "AUTH_STALE" |
||||
StatusNames[AUTH_ERROR] = "AUTH_ERROR" |
||||
StatusNames[AUTH_CONTINUE] = "AUTH_CONTINUE" |
||||
StatusNames[ERANGE] = "ERANGE" |
||||
StatusNames[ROLLBACK] = "ROLLBACK" |
||||
StatusNames[EACCESS] = "EACCESS" |
||||
StatusNames[NOT_INITIALIZED] = "NOT_INITIALIZED" |
||||
StatusNames[UNKNOWN_COMMAND] = "UNKNOWN_COMMAND" |
||||
StatusNames[ENOMEM] = "ENOMEM" |
||||
StatusNames[NOT_SUPPORTED] = "NOT_SUPPORTED" |
||||
StatusNames[EINTERNAL] = "EINTERNAL" |
||||
StatusNames[EBUSY] = "EBUSY" |
||||
StatusNames[TMPFAIL] = "TMPFAIL" |
||||
StatusNames[SUBDOC_PATH_NOT_FOUND] = "SUBDOC_PATH_NOT_FOUND" |
||||
StatusNames[SUBDOC_BAD_MULTI] = "SUBDOC_BAD_MULTI" |
||||
|
||||
} |
||||
|
||||
// String an op code.
|
||||
func (o CommandCode) String() (rv string) { |
||||
rv = CommandNames[o] |
||||
if rv == "" { |
||||
rv = fmt.Sprintf("0x%02x", int(o)) |
||||
} |
||||
return rv |
||||
} |
||||
|
||||
// String an op code.
|
||||
func (s Status) String() (rv string) { |
||||
rv = StatusNames[s] |
||||
if rv == "" { |
||||
rv = fmt.Sprintf("0x%02x", int(s)) |
||||
} |
||||
return rv |
||||
} |
||||
|
||||
// IsQuiet will return true if a command is a "quiet" command.
|
||||
func (o CommandCode) IsQuiet() bool { |
||||
switch o { |
||||
case GETQ, |
||||
GETKQ, |
||||
SETQ, |
||||
ADDQ, |
||||
REPLACEQ, |
||||
DELETEQ, |
||||
INCREMENTQ, |
||||
DECREMENTQ, |
||||
QUITQ, |
||||
FLUSHQ, |
||||
APPENDQ, |
||||
PREPENDQ, |
||||
RSETQ, |
||||
RAPPENDQ, |
||||
RPREPENDQ, |
||||
RDELETEQ, |
||||
RINCRQ, |
||||
RDECRQ: |
||||
return true |
||||
} |
||||
return false |
||||
} |
||||
@ -0,0 +1,197 @@ |
||||
package gomemcached |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
"fmt" |
||||
"io" |
||||
) |
||||
|
||||
// The maximum reasonable body length to expect.
|
||||
// Anything larger than this will result in an error.
|
||||
// The current limit, 20MB, is the size limit supported by ep-engine.
|
||||
var MaxBodyLen = int(20 * 1024 * 1024) |
||||
|
||||
// MCRequest is memcached Request
|
||||
type MCRequest struct { |
||||
// The command being issued
|
||||
Opcode CommandCode |
||||
// The CAS (if applicable, or 0)
|
||||
Cas uint64 |
||||
// An opaque value to be returned with this request
|
||||
Opaque uint32 |
||||
// The vbucket to which this command belongs
|
||||
VBucket uint16 |
||||
// Command extras, key, and body
|
||||
Extras, Key, Body, ExtMeta []byte |
||||
// Datatype identifier
|
||||
DataType uint8 |
||||
} |
||||
|
||||
// Size gives the number of bytes this request requires.
|
||||
func (req *MCRequest) Size() int { |
||||
return HDR_LEN + len(req.Extras) + len(req.Key) + len(req.Body) + len(req.ExtMeta) |
||||
} |
||||
|
||||
// A debugging string representation of this request
|
||||
func (req MCRequest) String() string { |
||||
return fmt.Sprintf("{MCRequest opcode=%s, bodylen=%d, key='%s'}", |
||||
req.Opcode, len(req.Body), req.Key) |
||||
} |
||||
|
||||
func (req *MCRequest) fillHeaderBytes(data []byte) int { |
||||
|
||||
pos := 0 |
||||
data[pos] = REQ_MAGIC |
||||
pos++ |
||||
data[pos] = byte(req.Opcode) |
||||
pos++ |
||||
binary.BigEndian.PutUint16(data[pos:pos+2], |
||||
uint16(len(req.Key))) |
||||
pos += 2 |
||||
|
||||
// 4
|
||||
data[pos] = byte(len(req.Extras)) |
||||
pos++ |
||||
// Data type
|
||||
if req.DataType != 0 { |
||||
data[pos] = byte(req.DataType) |
||||
} |
||||
pos++ |
||||
binary.BigEndian.PutUint16(data[pos:pos+2], req.VBucket) |
||||
pos += 2 |
||||
|
||||
// 8
|
||||
binary.BigEndian.PutUint32(data[pos:pos+4], |
||||
uint32(len(req.Body)+len(req.Key)+len(req.Extras)+len(req.ExtMeta))) |
||||
pos += 4 |
||||
|
||||
// 12
|
||||
binary.BigEndian.PutUint32(data[pos:pos+4], req.Opaque) |
||||
pos += 4 |
||||
|
||||
// 16
|
||||
if req.Cas != 0 { |
||||
binary.BigEndian.PutUint64(data[pos:pos+8], req.Cas) |
||||
} |
||||
pos += 8 |
||||
|
||||
if len(req.Extras) > 0 { |
||||
copy(data[pos:pos+len(req.Extras)], req.Extras) |
||||
pos += len(req.Extras) |
||||
} |
||||
|
||||
if len(req.Key) > 0 { |
||||
copy(data[pos:pos+len(req.Key)], req.Key) |
||||
pos += len(req.Key) |
||||
} |
||||
|
||||
return pos |
||||
} |
||||
|
||||
// HeaderBytes will return the wire representation of the request header
|
||||
// (with the extras and key).
|
||||
func (req *MCRequest) HeaderBytes() []byte { |
||||
data := make([]byte, HDR_LEN+len(req.Extras)+len(req.Key)) |
||||
|
||||
req.fillHeaderBytes(data) |
||||
|
||||
return data |
||||
} |
||||
|
||||
// Bytes will return the wire representation of this request.
|
||||
func (req *MCRequest) Bytes() []byte { |
||||
data := make([]byte, req.Size()) |
||||
|
||||
pos := req.fillHeaderBytes(data) |
||||
|
||||
if len(req.Body) > 0 { |
||||
copy(data[pos:pos+len(req.Body)], req.Body) |
||||
} |
||||
|
||||
if len(req.ExtMeta) > 0 { |
||||
copy(data[pos+len(req.Body):pos+len(req.Body)+len(req.ExtMeta)], req.ExtMeta) |
||||
} |
||||
|
||||
return data |
||||
} |
||||
|
||||
// Transmit will send this request message across a writer.
|
||||
func (req *MCRequest) Transmit(w io.Writer) (n int, err error) { |
||||
if len(req.Body) < 128 { |
||||
n, err = w.Write(req.Bytes()) |
||||
} else { |
||||
n, err = w.Write(req.HeaderBytes()) |
||||
if err == nil { |
||||
m := 0 |
||||
m, err = w.Write(req.Body) |
||||
n += m |
||||
} |
||||
} |
||||
return |
||||
} |
||||
|
||||
// Receive will fill this MCRequest with the data from a reader.
|
||||
func (req *MCRequest) Receive(r io.Reader, hdrBytes []byte) (int, error) { |
||||
if len(hdrBytes) < HDR_LEN { |
||||
hdrBytes = []byte{ |
||||
0, 0, 0, 0, 0, 0, 0, 0, |
||||
0, 0, 0, 0, 0, 0, 0, 0, |
||||
0, 0, 0, 0, 0, 0, 0, 0} |
||||
} |
||||
n, err := io.ReadFull(r, hdrBytes) |
||||
if err != nil { |
||||
return n, err |
||||
} |
||||
|
||||
if hdrBytes[0] != RES_MAGIC && hdrBytes[0] != REQ_MAGIC { |
||||
return n, fmt.Errorf("bad magic: 0x%02x", hdrBytes[0]) |
||||
} |
||||
|
||||
klen := int(binary.BigEndian.Uint16(hdrBytes[2:])) |
||||
elen := int(hdrBytes[4]) |
||||
// Data type at 5
|
||||
req.DataType = uint8(hdrBytes[5]) |
||||
|
||||
req.Opcode = CommandCode(hdrBytes[1]) |
||||
// Vbucket at 6:7
|
||||
req.VBucket = binary.BigEndian.Uint16(hdrBytes[6:]) |
||||
totalBodyLen := int(binary.BigEndian.Uint32(hdrBytes[8:])) |
||||
|
||||
req.Opaque = binary.BigEndian.Uint32(hdrBytes[12:]) |
||||
req.Cas = binary.BigEndian.Uint64(hdrBytes[16:]) |
||||
|
||||
if totalBodyLen > 0 { |
||||
buf := make([]byte, totalBodyLen) |
||||
m, err := io.ReadFull(r, buf) |
||||
n += m |
||||
if err == nil { |
||||
if req.Opcode >= TAP_MUTATION && |
||||
req.Opcode <= TAP_CHECKPOINT_END && |
||||
len(buf) > 1 { |
||||
// In these commands there is "engine private"
|
||||
// data at the end of the extras. The first 2
|
||||
// bytes of extra data give its length.
|
||||
elen += int(binary.BigEndian.Uint16(buf)) |
||||
} |
||||
|
||||
req.Extras = buf[0:elen] |
||||
req.Key = buf[elen : klen+elen] |
||||
|
||||
// get the length of extended metadata
|
||||
extMetaLen := 0 |
||||
if elen > 29 { |
||||
extMetaLen = int(binary.BigEndian.Uint16(req.Extras[28:30])) |
||||
} |
||||
|
||||
bodyLen := totalBodyLen - klen - elen - extMetaLen |
||||
if bodyLen > MaxBodyLen { |
||||
return n, fmt.Errorf("%d is too big (max %d)", |
||||
bodyLen, MaxBodyLen) |
||||
} |
||||
|
||||
req.Body = buf[klen+elen : klen+elen+bodyLen] |
||||
req.ExtMeta = buf[klen+elen+bodyLen:] |
||||
} |
||||
} |
||||
return n, err |
||||
} |
||||
@ -0,0 +1,267 @@ |
||||
package gomemcached |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
"fmt" |
||||
"io" |
||||
"sync" |
||||
) |
||||
|
||||
// MCResponse is memcached response
|
||||
type MCResponse struct { |
||||
// The command opcode of the command that sent the request
|
||||
Opcode CommandCode |
||||
// The status of the response
|
||||
Status Status |
||||
// The opaque sent in the request
|
||||
Opaque uint32 |
||||
// The CAS identifier (if applicable)
|
||||
Cas uint64 |
||||
// Extras, key, and body for this response
|
||||
Extras, Key, Body []byte |
||||
// If true, this represents a fatal condition and we should hang up
|
||||
Fatal bool |
||||
// Datatype identifier
|
||||
DataType uint8 |
||||
} |
||||
|
||||
// A debugging string representation of this response
|
||||
func (res MCResponse) String() string { |
||||
return fmt.Sprintf("{MCResponse status=%v keylen=%d, extralen=%d, bodylen=%d}", |
||||
res.Status, len(res.Key), len(res.Extras), len(res.Body)) |
||||
} |
||||
|
||||
// Response as an error.
|
||||
func (res *MCResponse) Error() string { |
||||
return fmt.Sprintf("MCResponse status=%v, opcode=%v, opaque=%v, msg: %s", |
||||
res.Status, res.Opcode, res.Opaque, string(res.Body)) |
||||
} |
||||
|
||||
func errStatus(e error) Status { |
||||
status := Status(0xffff) |
||||
if res, ok := e.(*MCResponse); ok { |
||||
status = res.Status |
||||
} |
||||
return status |
||||
} |
||||
|
||||
// IsNotFound is true if this error represents a "not found" response.
|
||||
func IsNotFound(e error) bool { |
||||
return errStatus(e) == KEY_ENOENT |
||||
} |
||||
|
||||
// IsFatal is false if this error isn't believed to be fatal to a connection.
|
||||
func IsFatal(e error) bool { |
||||
if e == nil { |
||||
return false |
||||
} |
||||
_, ok := isFatal[errStatus(e)] |
||||
if ok { |
||||
return true |
||||
} |
||||
return false |
||||
} |
||||
|
||||
// Size is number of bytes this response consumes on the wire.
|
||||
func (res *MCResponse) Size() int { |
||||
return HDR_LEN + len(res.Extras) + len(res.Key) + len(res.Body) |
||||
} |
||||
|
||||
func (res *MCResponse) fillHeaderBytes(data []byte) int { |
||||
pos := 0 |
||||
data[pos] = RES_MAGIC |
||||
pos++ |
||||
data[pos] = byte(res.Opcode) |
||||
pos++ |
||||
binary.BigEndian.PutUint16(data[pos:pos+2], |
||||
uint16(len(res.Key))) |
||||
pos += 2 |
||||
|
||||
// 4
|
||||
data[pos] = byte(len(res.Extras)) |
||||
pos++ |
||||
// Data type
|
||||
if res.DataType != 0 { |
||||
data[pos] = byte(res.DataType) |
||||
} else { |
||||
data[pos] = 0 |
||||
} |
||||
pos++ |
||||
binary.BigEndian.PutUint16(data[pos:pos+2], uint16(res.Status)) |
||||
pos += 2 |
||||
|
||||
// 8
|
||||
binary.BigEndian.PutUint32(data[pos:pos+4], |
||||
uint32(len(res.Body)+len(res.Key)+len(res.Extras))) |
||||
pos += 4 |
||||
|
||||
// 12
|
||||
binary.BigEndian.PutUint32(data[pos:pos+4], res.Opaque) |
||||
pos += 4 |
||||
|
||||
// 16
|
||||
binary.BigEndian.PutUint64(data[pos:pos+8], res.Cas) |
||||
pos += 8 |
||||
|
||||
if len(res.Extras) > 0 { |
||||
copy(data[pos:pos+len(res.Extras)], res.Extras) |
||||
pos += len(res.Extras) |
||||
} |
||||
|
||||
if len(res.Key) > 0 { |
||||
copy(data[pos:pos+len(res.Key)], res.Key) |
||||
pos += len(res.Key) |
||||
} |
||||
|
||||
return pos |
||||
} |
||||
|
||||
// HeaderBytes will get just the header bytes for this response.
|
||||
func (res *MCResponse) HeaderBytes() []byte { |
||||
data := make([]byte, HDR_LEN+len(res.Extras)+len(res.Key)) |
||||
|
||||
res.fillHeaderBytes(data) |
||||
|
||||
return data |
||||
} |
||||
|
||||
// Bytes will return the actual bytes transmitted for this response.
|
||||
func (res *MCResponse) Bytes() []byte { |
||||
data := make([]byte, res.Size()) |
||||
|
||||
pos := res.fillHeaderBytes(data) |
||||
|
||||
copy(data[pos:pos+len(res.Body)], res.Body) |
||||
|
||||
return data |
||||
} |
||||
|
||||
// Transmit will send this response message across a writer.
|
||||
func (res *MCResponse) Transmit(w io.Writer) (n int, err error) { |
||||
if len(res.Body) < 128 { |
||||
n, err = w.Write(res.Bytes()) |
||||
} else { |
||||
n, err = w.Write(res.HeaderBytes()) |
||||
if err == nil { |
||||
m := 0 |
||||
m, err = w.Write(res.Body) |
||||
m += n |
||||
} |
||||
} |
||||
return |
||||
} |
||||
|
||||
// Receive will fill this MCResponse with the data from this reader.
|
||||
func (res *MCResponse) Receive(r io.Reader, hdrBytes []byte) (n int, err error) { |
||||
if len(hdrBytes) < HDR_LEN { |
||||
hdrBytes = []byte{ |
||||
0, 0, 0, 0, 0, 0, 0, 0, |
||||
0, 0, 0, 0, 0, 0, 0, 0, |
||||
0, 0, 0, 0, 0, 0, 0, 0} |
||||
} |
||||
n, err = io.ReadFull(r, hdrBytes) |
||||
if err != nil { |
||||
return n, err |
||||
} |
||||
|
||||
if hdrBytes[0] != RES_MAGIC && hdrBytes[0] != REQ_MAGIC { |
||||
return n, fmt.Errorf("bad magic: 0x%02x", hdrBytes[0]) |
||||
} |
||||
|
||||
klen := int(binary.BigEndian.Uint16(hdrBytes[2:4])) |
||||
elen := int(hdrBytes[4]) |
||||
|
||||
res.Opcode = CommandCode(hdrBytes[1]) |
||||
res.DataType = uint8(hdrBytes[5]) |
||||
res.Status = Status(binary.BigEndian.Uint16(hdrBytes[6:8])) |
||||
res.Opaque = binary.BigEndian.Uint32(hdrBytes[12:16]) |
||||
res.Cas = binary.BigEndian.Uint64(hdrBytes[16:24]) |
||||
|
||||
bodyLen := int(binary.BigEndian.Uint32(hdrBytes[8:12])) - (klen + elen) |
||||
|
||||
//defer function to debug the panic seen with MB-15557
|
||||
defer func() { |
||||
if e := recover(); e != nil { |
||||
err = fmt.Errorf(`Panic in Receive. Response %v \n |
||||
key len %v extra len %v bodylen %v`, res, klen, elen, bodyLen) |
||||
} |
||||
}() |
||||
|
||||
buf := make([]byte, klen+elen+bodyLen) |
||||
m, err := io.ReadFull(r, buf) |
||||
if err == nil { |
||||
res.Extras = buf[0:elen] |
||||
res.Key = buf[elen : klen+elen] |
||||
res.Body = buf[klen+elen:] |
||||
} |
||||
|
||||
return n + m, err |
||||
} |
||||
|
||||
type MCResponsePool struct { |
||||
pool *sync.Pool |
||||
} |
||||
|
||||
func NewMCResponsePool() *MCResponsePool { |
||||
rv := &MCResponsePool{ |
||||
pool: &sync.Pool{ |
||||
New: func() interface{} { |
||||
return &MCResponse{} |
||||
}, |
||||
}, |
||||
} |
||||
|
||||
return rv |
||||
} |
||||
|
||||
func (this *MCResponsePool) Get() *MCResponse { |
||||
return this.pool.Get().(*MCResponse) |
||||
} |
||||
|
||||
func (this *MCResponsePool) Put(r *MCResponse) { |
||||
if r == nil { |
||||
return |
||||
} |
||||
|
||||
r.Extras = nil |
||||
r.Key = nil |
||||
r.Body = nil |
||||
r.Fatal = false |
||||
|
||||
this.pool.Put(r) |
||||
} |
||||
|
||||
type StringMCResponsePool struct { |
||||
pool *sync.Pool |
||||
size int |
||||
} |
||||
|
||||
func NewStringMCResponsePool(size int) *StringMCResponsePool { |
||||
rv := &StringMCResponsePool{ |
||||
pool: &sync.Pool{ |
||||
New: func() interface{} { |
||||
return make(map[string]*MCResponse, size) |
||||
}, |
||||
}, |
||||
size: size, |
||||
} |
||||
|
||||
return rv |
||||
} |
||||
|
||||
func (this *StringMCResponsePool) Get() map[string]*MCResponse { |
||||
return this.pool.Get().(map[string]*MCResponse) |
||||
} |
||||
|
||||
func (this *StringMCResponsePool) Put(m map[string]*MCResponse) { |
||||
if m == nil || len(m) > 2*this.size { |
||||
return |
||||
} |
||||
|
||||
for k := range m { |
||||
m[k] = nil |
||||
delete(m, k) |
||||
} |
||||
|
||||
this.pool.Put(m) |
||||
} |
||||
@ -0,0 +1,168 @@ |
||||
package gomemcached |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/binary" |
||||
"fmt" |
||||
"io" |
||||
"io/ioutil" |
||||
"strings" |
||||
) |
||||
|
||||
type TapConnectFlag uint32 |
||||
|
||||
// Tap connect option flags
|
||||
const ( |
||||
BACKFILL = TapConnectFlag(0x01) |
||||
DUMP = TapConnectFlag(0x02) |
||||
LIST_VBUCKETS = TapConnectFlag(0x04) |
||||
TAKEOVER_VBUCKETS = TapConnectFlag(0x08) |
||||
SUPPORT_ACK = TapConnectFlag(0x10) |
||||
REQUEST_KEYS_ONLY = TapConnectFlag(0x20) |
||||
CHECKPOINT = TapConnectFlag(0x40) |
||||
REGISTERED_CLIENT = TapConnectFlag(0x80) |
||||
FIX_FLAG_BYTEORDER = TapConnectFlag(0x100) |
||||
) |
||||
|
||||
// Tap opaque event subtypes
|
||||
const ( |
||||
TAP_OPAQUE_ENABLE_AUTO_NACK = 0 |
||||
TAP_OPAQUE_INITIAL_VBUCKET_STREAM = 1 |
||||
TAP_OPAQUE_ENABLE_CHECKPOINT_SYNC = 2 |
||||
TAP_OPAQUE_CLOSE_TAP_STREAM = 7 |
||||
TAP_OPAQUE_CLOSE_BACKFILL = 8 |
||||
) |
||||
|
||||
// Tap item flags
|
||||
const ( |
||||
TAP_ACK = 1 |
||||
TAP_NO_VALUE = 2 |
||||
TAP_FLAG_NETWORK_BYTE_ORDER = 4 |
||||
) |
||||
|
||||
// TapConnectFlagNames for TapConnectFlag
|
||||
var TapConnectFlagNames = map[TapConnectFlag]string{ |
||||
BACKFILL: "BACKFILL", |
||||
DUMP: "DUMP", |
||||
LIST_VBUCKETS: "LIST_VBUCKETS", |
||||
TAKEOVER_VBUCKETS: "TAKEOVER_VBUCKETS", |
||||
SUPPORT_ACK: "SUPPORT_ACK", |
||||
REQUEST_KEYS_ONLY: "REQUEST_KEYS_ONLY", |
||||
CHECKPOINT: "CHECKPOINT", |
||||
REGISTERED_CLIENT: "REGISTERED_CLIENT", |
||||
FIX_FLAG_BYTEORDER: "FIX_FLAG_BYTEORDER", |
||||
} |
||||
|
||||
// TapItemParser is a function to parse a single tap extra.
|
||||
type TapItemParser func(io.Reader) (interface{}, error) |
||||
|
||||
// TapParseUint64 is a function to parse a single tap uint64.
|
||||
func TapParseUint64(r io.Reader) (interface{}, error) { |
||||
var rv uint64 |
||||
err := binary.Read(r, binary.BigEndian, &rv) |
||||
return rv, err |
||||
} |
||||
|
||||
// TapParseUint16 is a function to parse a single tap uint16.
|
||||
func TapParseUint16(r io.Reader) (interface{}, error) { |
||||
var rv uint16 |
||||
err := binary.Read(r, binary.BigEndian, &rv) |
||||
return rv, err |
||||
} |
||||
|
||||
// TapParseBool is a function to parse a single tap boolean.
|
||||
func TapParseBool(r io.Reader) (interface{}, error) { |
||||
return true, nil |
||||
} |
||||
|
||||
// TapParseVBList parses a list of vBucket numbers as []uint16.
|
||||
func TapParseVBList(r io.Reader) (interface{}, error) { |
||||
num, err := TapParseUint16(r) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
n := int(num.(uint16)) |
||||
|
||||
rv := make([]uint16, n) |
||||
for i := 0; i < n; i++ { |
||||
x, err := TapParseUint16(r) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
rv[i] = x.(uint16) |
||||
} |
||||
|
||||
return rv, err |
||||
} |
||||
|
||||
// TapFlagParsers parser functions for TAP fields.
|
||||
var TapFlagParsers = map[TapConnectFlag]TapItemParser{ |
||||
BACKFILL: TapParseUint64, |
||||
LIST_VBUCKETS: TapParseVBList, |
||||
} |
||||
|
||||
// SplitFlags will split the ORed flags into the individual bit flags.
|
||||
func (f TapConnectFlag) SplitFlags() []TapConnectFlag { |
||||
rv := []TapConnectFlag{} |
||||
for i := uint32(1); f != 0; i = i << 1 { |
||||
if uint32(f)&i == i { |
||||
rv = append(rv, TapConnectFlag(i)) |
||||
} |
||||
f = TapConnectFlag(uint32(f) & (^i)) |
||||
} |
||||
return rv |
||||
} |
||||
|
||||
func (f TapConnectFlag) String() string { |
||||
parts := []string{} |
||||
for _, x := range f.SplitFlags() { |
||||
p := TapConnectFlagNames[x] |
||||
if p == "" { |
||||
p = fmt.Sprintf("0x%x", int(x)) |
||||
} |
||||
parts = append(parts, p) |
||||
} |
||||
return strings.Join(parts, "|") |
||||
} |
||||
|
||||
type TapConnect struct { |
||||
Flags map[TapConnectFlag]interface{} |
||||
RemainingBody []byte |
||||
Name string |
||||
} |
||||
|
||||
// ParseTapCommands parse the tap request into the interesting bits we may
|
||||
// need to do something with.
|
||||
func (req *MCRequest) ParseTapCommands() (TapConnect, error) { |
||||
rv := TapConnect{ |
||||
Flags: map[TapConnectFlag]interface{}{}, |
||||
Name: string(req.Key), |
||||
} |
||||
|
||||
if len(req.Extras) < 4 { |
||||
return rv, fmt.Errorf("not enough extra bytes: %x", req.Extras) |
||||
} |
||||
|
||||
flags := TapConnectFlag(binary.BigEndian.Uint32(req.Extras)) |
||||
|
||||
r := bytes.NewReader(req.Body) |
||||
|
||||
for _, f := range flags.SplitFlags() { |
||||
fun := TapFlagParsers[f] |
||||
if fun == nil { |
||||
fun = TapParseBool |
||||
} |
||||
|
||||
val, err := fun(r) |
||||
if err != nil { |
||||
return rv, err |
||||
} |
||||
|
||||
rv.Flags[f] = val |
||||
} |
||||
|
||||
var err error |
||||
rv.RemainingBody, err = ioutil.ReadAll(r) |
||||
|
||||
return rv, err |
||||
} |
||||
@ -0,0 +1,47 @@ |
||||
COUCHBASE INC. COMMUNITY EDITION LICENSE AGREEMENT |
||||
|
||||
IMPORTANT-READ CAREFULLY: BY CLICKING THE "I ACCEPT" BOX OR INSTALLING, |
||||
DOWNLOADING OR OTHERWISE USING THIS SOFTWARE AND ANY ASSOCIATED |
||||
DOCUMENTATION, YOU, ON BEHALF OF YOURSELF OR AS AN AUTHORIZED |
||||
REPRESENTATIVE ON BEHALF OF AN ENTITY ("LICENSEE") AGREE TO ALL THE |
||||
TERMS OF THIS COMMUNITY EDITION LICENSE AGREEMENT (THE "AGREEMENT") |
||||
REGARDING YOUR USE OF THE SOFTWARE. YOU REPRESENT AND WARRANT THAT YOU |
||||
HAVE FULL LEGAL AUTHORITY TO BIND THE LICENSEE TO THIS AGREEMENT. IF YOU |
||||
DO NOT AGREE WITH ALL OF THESE TERMS, DO NOT SELECT THE "I ACCEPT" BOX |
||||
AND DO NOT INSTALL, DOWNLOAD OR OTHERWISE USE THE SOFTWARE. THE |
||||
EFFECTIVE DATE OF THIS AGREEMENT IS THE DATE ON WHICH YOU CLICK "I |
||||
ACCEPT" OR OTHERWISE INSTALL, DOWNLOAD OR USE THE SOFTWARE. |
||||
|
||||
1. License Grant. Couchbase Inc. hereby grants Licensee, free of charge, |
||||
the non-exclusive right to use, copy, merge, publish, distribute, |
||||
sublicense, and/or sell copies of the Software, and to permit persons to |
||||
whom the Software is furnished to do so, subject to Licensee including |
||||
the following copyright notice in all copies or substantial portions of |
||||
the Software: |
||||
|
||||
Couchbase (r) http://www.Couchbase.com Copyright 2016 Couchbase, Inc. |
||||
|
||||
As used in this Agreement, "Software" means the object code version of |
||||
the applicable elastic data management server software provided by |
||||
Couchbase Inc. |
||||
|
||||
2. Restrictions. Licensee will not reverse engineer, disassemble, or |
||||
decompile the Software (except to the extent such restrictions are |
||||
prohibited by law). |
||||
|
||||
3. Support. Couchbase, Inc. will provide Licensee with access to, and |
||||
use of, the Couchbase, Inc. support forum available at the following |
||||
URL: http://www.couchbase.org/forums/. Couchbase, Inc. may, at its |
||||
discretion, modify, suspend or terminate support at any time upon notice |
||||
to Licensee. |
||||
|
||||
4. Warranty Disclaimer and Limitation of Liability. THE SOFTWARE IS |
||||
PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, |
||||
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS |
||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
||||
COUCHBASE INC. OR THE AUTHORS OR COPYRIGHT HOLDERS IN THE SOFTWARE BE |
||||
LIABLE FOR ANY CLAIM, DAMAGES (IINCLUDING, WITHOUT LIMITATION, DIRECT, |
||||
INDIRECT OR CONSEQUENTIAL DAMAGES) OR OTHER LIABILITY, WHETHER IN AN |
||||
ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
||||
SOFTWARE. |
||||
@ -0,0 +1,481 @@ |
||||
// Copyright (c) 2016 Couchbase, Inc.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
|
||||
// except in compliance with the License. You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the
|
||||
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
// either express or implied. See the License for the specific language governing permissions
|
||||
// and limitations under the License.
|
||||
|
||||
package logging |
||||
|
||||
import ( |
||||
"os" |
||||
"runtime" |
||||
"strings" |
||||
"sync" |
||||
) |
||||
|
||||
type Level int |
||||
|
||||
const ( |
||||
NONE = Level(iota) // Disable all logging
|
||||
FATAL // System is in severe error state and has to abort
|
||||
SEVERE // System is in severe error state and cannot recover reliably
|
||||
ERROR // System is in error state but can recover and continue reliably
|
||||
WARN // System approaching error state, or is in a correct but undesirable state
|
||||
INFO // System-level events and status, in correct states
|
||||
REQUEST // Request-level events, with request-specific rlevel
|
||||
TRACE // Trace detailed system execution, e.g. function entry / exit
|
||||
DEBUG // Debug
|
||||
) |
||||
|
||||
type LogEntryFormatter int |
||||
|
||||
const ( |
||||
TEXTFORMATTER = LogEntryFormatter(iota) |
||||
JSONFORMATTER |
||||
KVFORMATTER |
||||
) |
||||
|
||||
func (level Level) String() string { |
||||
return _LEVEL_NAMES[level] |
||||
} |
||||
|
||||
var _LEVEL_NAMES = []string{ |
||||
DEBUG: "DEBUG", |
||||
TRACE: "TRACE", |
||||
REQUEST: "REQUEST", |
||||
INFO: "INFO", |
||||
WARN: "WARN", |
||||
ERROR: "ERROR", |
||||
SEVERE: "SEVERE", |
||||
FATAL: "FATAL", |
||||
NONE: "NONE", |
||||
} |
||||
|
||||
var _LEVEL_MAP = map[string]Level{ |
||||
"debug": DEBUG, |
||||
"trace": TRACE, |
||||
"request": REQUEST, |
||||
"info": INFO, |
||||
"warn": WARN, |
||||
"error": ERROR, |
||||
"severe": SEVERE, |
||||
"fatal": FATAL, |
||||
"none": NONE, |
||||
} |
||||
|
||||
func ParseLevel(name string) (level Level, ok bool) { |
||||
level, ok = _LEVEL_MAP[strings.ToLower(name)] |
||||
return |
||||
} |
||||
|
||||
/* |
||||
|
||||
Pair supports logging of key-value pairs. Keys beginning with _ are |
||||
reserved for the logger, e.g. _time, _level, _msg, and _rlevel. The |
||||
Pair APIs are designed to avoid heap allocation and garbage |
||||
collection. |
||||
|
||||
*/ |
||||
type Pairs []Pair |
||||
type Pair struct { |
||||
Name string |
||||
Value interface{} |
||||
} |
||||
|
||||
/* |
||||
|
||||
Map allows key-value pairs to be specified using map literals or data |
||||
structures. For example: |
||||
|
||||
Errorm(msg, Map{...}) |
||||
|
||||
Map incurs heap allocation and garbage collection, so the Pair APIs |
||||
should be preferred. |
||||
|
||||
*/ |
||||
type Map map[string]interface{} |
||||
|
||||
// Logger provides a common interface for logging libraries
|
||||
type Logger interface { |
||||
/* |
||||
These APIs write all the given pairs in addition to standard logger keys. |
||||
*/ |
||||
Logp(level Level, msg string, kv ...Pair) |
||||
|
||||
Debugp(msg string, kv ...Pair) |
||||
|
||||
Tracep(msg string, kv ...Pair) |
||||
|
||||
Requestp(rlevel Level, msg string, kv ...Pair) |
||||
|
||||
Infop(msg string, kv ...Pair) |
||||
|
||||
Warnp(msg string, kv ...Pair) |
||||
|
||||
Errorp(msg string, kv ...Pair) |
||||
|
||||
Severep(msg string, kv ...Pair) |
||||
|
||||
Fatalp(msg string, kv ...Pair) |
||||
|
||||
/* |
||||
These APIs write the fields in the given kv Map in addition to standard logger keys. |
||||
*/ |
||||
Logm(level Level, msg string, kv Map) |
||||
|
||||
Debugm(msg string, kv Map) |
||||
|
||||
Tracem(msg string, kv Map) |
||||
|
||||
Requestm(rlevel Level, msg string, kv Map) |
||||
|
||||
Infom(msg string, kv Map) |
||||
|
||||
Warnm(msg string, kv Map) |
||||
|
||||
Errorm(msg string, kv Map) |
||||
|
||||
Severem(msg string, kv Map) |
||||
|
||||
Fatalm(msg string, kv Map) |
||||
|
||||
/* |
||||
|
||||
These APIs only write _msg, _time, _level, and other logger keys. If |
||||
the msg contains other fields, use the Pair or Map APIs instead. |
||||
|
||||
*/ |
||||
Logf(level Level, fmt string, args ...interface{}) |
||||
|
||||
Debugf(fmt string, args ...interface{}) |
||||
|
||||
Tracef(fmt string, args ...interface{}) |
||||
|
||||
Requestf(rlevel Level, fmt string, args ...interface{}) |
||||
|
||||
Infof(fmt string, args ...interface{}) |
||||
|
||||
Warnf(fmt string, args ...interface{}) |
||||
|
||||
Errorf(fmt string, args ...interface{}) |
||||
|
||||
Severef(fmt string, args ...interface{}) |
||||
|
||||
Fatalf(fmt string, args ...interface{}) |
||||
|
||||
/* |
||||
These APIs control the logging level |
||||
*/ |
||||
|
||||
SetLevel(Level) // Set the logging level
|
||||
|
||||
Level() Level // Get the current logging level
|
||||
} |
||||
|
||||
var logger Logger = nil |
||||
var curLevel Level = DEBUG // initially set to never skip
|
||||
|
||||
var loggerMutex sync.RWMutex |
||||
|
||||
// All the methods below first acquire the mutex (mostly in exclusive mode)
|
||||
// and only then check if logging at the current level is enabled.
|
||||
// This introduces a fair bottleneck for those log entries that should be
|
||||
// skipped (the majority, at INFO or below levels)
|
||||
// We try to predict here if we should lock the mutex at all by caching
|
||||
// the current log level: while dynamically changing logger, there might
|
||||
// be the odd entry skipped as the new level is cached.
|
||||
// Since we seem to never change the logger, this is not an issue.
|
||||
func skipLogging(level Level) bool { |
||||
if logger == nil { |
||||
return true |
||||
} |
||||
return level > curLevel |
||||
} |
||||
|
||||
func SetLogger(newLogger Logger) { |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger = newLogger |
||||
if logger == nil { |
||||
curLevel = NONE |
||||
} else { |
||||
curLevel = newLogger.Level() |
||||
} |
||||
} |
||||
|
||||
func Logp(level Level, msg string, kv ...Pair) { |
||||
if skipLogging(level) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Logp(level, msg, kv...) |
||||
} |
||||
|
||||
func Debugp(msg string, kv ...Pair) { |
||||
if skipLogging(DEBUG) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Debugp(msg, kv...) |
||||
} |
||||
|
||||
func Tracep(msg string, kv ...Pair) { |
||||
if skipLogging(TRACE) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Tracep(msg, kv...) |
||||
} |
||||
|
||||
func Requestp(rlevel Level, msg string, kv ...Pair) { |
||||
if skipLogging(REQUEST) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Requestp(rlevel, msg, kv...) |
||||
} |
||||
|
||||
func Infop(msg string, kv ...Pair) { |
||||
if skipLogging(INFO) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Infop(msg, kv...) |
||||
} |
||||
|
||||
func Warnp(msg string, kv ...Pair) { |
||||
if skipLogging(WARN) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Warnp(msg, kv...) |
||||
} |
||||
|
||||
func Errorp(msg string, kv ...Pair) { |
||||
if skipLogging(ERROR) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Errorp(msg, kv...) |
||||
} |
||||
|
||||
func Severep(msg string, kv ...Pair) { |
||||
if skipLogging(SEVERE) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Severep(msg, kv...) |
||||
} |
||||
|
||||
func Fatalp(msg string, kv ...Pair) { |
||||
if skipLogging(FATAL) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Fatalp(msg, kv...) |
||||
} |
||||
|
||||
func Logm(level Level, msg string, kv Map) { |
||||
if skipLogging(level) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Logm(level, msg, kv) |
||||
} |
||||
|
||||
func Debugm(msg string, kv Map) { |
||||
if skipLogging(DEBUG) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Debugm(msg, kv) |
||||
} |
||||
|
||||
func Tracem(msg string, kv Map) { |
||||
if skipLogging(TRACE) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Tracem(msg, kv) |
||||
} |
||||
|
||||
func Requestm(rlevel Level, msg string, kv Map) { |
||||
if skipLogging(REQUEST) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Requestm(rlevel, msg, kv) |
||||
} |
||||
|
||||
func Infom(msg string, kv Map) { |
||||
if skipLogging(INFO) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Infom(msg, kv) |
||||
} |
||||
|
||||
func Warnm(msg string, kv Map) { |
||||
if skipLogging(WARN) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Warnm(msg, kv) |
||||
} |
||||
|
||||
func Errorm(msg string, kv Map) { |
||||
if skipLogging(ERROR) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Errorm(msg, kv) |
||||
} |
||||
|
||||
func Severem(msg string, kv Map) { |
||||
if skipLogging(SEVERE) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Severem(msg, kv) |
||||
} |
||||
|
||||
func Fatalm(msg string, kv Map) { |
||||
if skipLogging(FATAL) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Fatalm(msg, kv) |
||||
} |
||||
|
||||
func Logf(level Level, fmt string, args ...interface{}) { |
||||
if skipLogging(level) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Logf(level, fmt, args...) |
||||
} |
||||
|
||||
func Debugf(fmt string, args ...interface{}) { |
||||
if skipLogging(DEBUG) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Debugf(fmt, args...) |
||||
} |
||||
|
||||
func Tracef(fmt string, args ...interface{}) { |
||||
if skipLogging(TRACE) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Tracef(fmt, args...) |
||||
} |
||||
|
||||
func Requestf(rlevel Level, fmt string, args ...interface{}) { |
||||
if skipLogging(REQUEST) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Requestf(rlevel, fmt, args...) |
||||
} |
||||
|
||||
func Infof(fmt string, args ...interface{}) { |
||||
if skipLogging(INFO) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Infof(fmt, args...) |
||||
} |
||||
|
||||
func Warnf(fmt string, args ...interface{}) { |
||||
if skipLogging(WARN) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Warnf(fmt, args...) |
||||
} |
||||
|
||||
func Errorf(fmt string, args ...interface{}) { |
||||
if skipLogging(ERROR) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Errorf(fmt, args...) |
||||
} |
||||
|
||||
func Severef(fmt string, args ...interface{}) { |
||||
if skipLogging(SEVERE) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Severef(fmt, args...) |
||||
} |
||||
|
||||
func Fatalf(fmt string, args ...interface{}) { |
||||
if skipLogging(FATAL) { |
||||
return |
||||
} |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Fatalf(fmt, args...) |
||||
} |
||||
|
||||
func SetLevel(level Level) { |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.SetLevel(level) |
||||
curLevel = level |
||||
} |
||||
|
||||
func LogLevel() Level { |
||||
loggerMutex.RLock() |
||||
defer loggerMutex.RUnlock() |
||||
return logger.Level() |
||||
} |
||||
|
||||
func Stackf(level Level, fmt string, args ...interface{}) { |
||||
if skipLogging(level) { |
||||
return |
||||
} |
||||
buf := make([]byte, 1<<16) |
||||
n := runtime.Stack(buf, false) |
||||
s := string(buf[0:n]) |
||||
loggerMutex.Lock() |
||||
defer loggerMutex.Unlock() |
||||
logger.Logf(level, fmt, args...) |
||||
logger.Logf(level, s) |
||||
} |
||||
|
||||
func init() { |
||||
logger = NewLogger(os.Stderr, INFO, TEXTFORMATTER) |
||||
SetLogger(logger) |
||||
} |
||||
@ -0,0 +1,318 @@ |
||||
// Copyright (c) 2016 Couchbase, Inc.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
|
||||
// except in compliance with the License. You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the
|
||||
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
// either express or implied. See the License for the specific language governing permissions
|
||||
// and limitations under the License.
|
||||
|
||||
package logging |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/json" |
||||
"fmt" |
||||
"io" |
||||
"log" |
||||
"time" |
||||
) |
||||
|
||||
type goLogger struct { |
||||
logger *log.Logger |
||||
level Level |
||||
entryFormatter formatter |
||||
} |
||||
|
||||
const ( |
||||
_LEVEL = "_level" |
||||
_MSG = "_msg" |
||||
_TIME = "_time" |
||||
_RLEVEL = "_rlevel" |
||||
) |
||||
|
||||
func NewLogger(out io.Writer, lvl Level, fmtLogging LogEntryFormatter) *goLogger { |
||||
logger := &goLogger{ |
||||
logger: log.New(out, "", 0), |
||||
level: lvl, |
||||
} |
||||
if fmtLogging == JSONFORMATTER { |
||||
logger.entryFormatter = &jsonFormatter{} |
||||
} else if fmtLogging == KVFORMATTER { |
||||
logger.entryFormatter = &keyvalueFormatter{} |
||||
} else { |
||||
logger.entryFormatter = &textFormatter{} |
||||
} |
||||
return logger |
||||
} |
||||
|
||||
func (gl *goLogger) Logp(level Level, msg string, kv ...Pair) { |
||||
if gl.logger == nil { |
||||
return |
||||
} |
||||
if level <= gl.level { |
||||
e := newLogEntry(msg, level) |
||||
copyPairs(e, kv) |
||||
gl.log(e) |
||||
} |
||||
} |
||||
|
||||
func (gl *goLogger) Debugp(msg string, kv ...Pair) { |
||||
gl.Logp(DEBUG, msg, kv...) |
||||
} |
||||
|
||||
func (gl *goLogger) Tracep(msg string, kv ...Pair) { |
||||
gl.Logp(TRACE, msg, kv...) |
||||
} |
||||
|
||||
func (gl *goLogger) Requestp(rlevel Level, msg string, kv ...Pair) { |
||||
if gl.logger == nil { |
||||
return |
||||
} |
||||
if REQUEST <= gl.level { |
||||
e := newLogEntry(msg, REQUEST) |
||||
e.Rlevel = rlevel |
||||
copyPairs(e, kv) |
||||
gl.log(e) |
||||
} |
||||
} |
||||
|
||||
func (gl *goLogger) Infop(msg string, kv ...Pair) { |
||||
gl.Logp(INFO, msg, kv...) |
||||
} |
||||
|
||||
func (gl *goLogger) Warnp(msg string, kv ...Pair) { |
||||
gl.Logp(WARN, msg, kv...) |
||||
} |
||||
|
||||
func (gl *goLogger) Errorp(msg string, kv ...Pair) { |
||||
gl.Logp(ERROR, msg, kv...) |
||||
} |
||||
|
||||
func (gl *goLogger) Severep(msg string, kv ...Pair) { |
||||
gl.Logp(SEVERE, msg, kv...) |
||||
} |
||||
|
||||
func (gl *goLogger) Fatalp(msg string, kv ...Pair) { |
||||
gl.Logp(FATAL, msg, kv...) |
||||
} |
||||
|
||||
func (gl *goLogger) Logm(level Level, msg string, kv Map) { |
||||
if gl.logger == nil { |
||||
return |
||||
} |
||||
if level <= gl.level { |
||||
e := newLogEntry(msg, level) |
||||
e.Data = kv |
||||
gl.log(e) |
||||
} |
||||
} |
||||
|
||||
func (gl *goLogger) Debugm(msg string, kv Map) { |
||||
gl.Logm(DEBUG, msg, kv) |
||||
} |
||||
|
||||
func (gl *goLogger) Tracem(msg string, kv Map) { |
||||
gl.Logm(TRACE, msg, kv) |
||||
} |
||||
|
||||
func (gl *goLogger) Requestm(rlevel Level, msg string, kv Map) { |
||||
if gl.logger == nil { |
||||
return |
||||
} |
||||
if REQUEST <= gl.level { |
||||
e := newLogEntry(msg, REQUEST) |
||||
e.Rlevel = rlevel |
||||
e.Data = kv |
||||
gl.log(e) |
||||
} |
||||
} |
||||
|
||||
func (gl *goLogger) Infom(msg string, kv Map) { |
||||
gl.Logm(INFO, msg, kv) |
||||
} |
||||
|
||||
func (gl *goLogger) Warnm(msg string, kv Map) { |
||||
gl.Logm(WARN, msg, kv) |
||||
} |
||||
|
||||
func (gl *goLogger) Errorm(msg string, kv Map) { |
||||
gl.Logm(ERROR, msg, kv) |
||||
} |
||||
|
||||
func (gl *goLogger) Severem(msg string, kv Map) { |
||||
gl.Logm(SEVERE, msg, kv) |
||||
} |
||||
|
||||
func (gl *goLogger) Fatalm(msg string, kv Map) { |
||||
gl.Logm(FATAL, msg, kv) |
||||
} |
||||
|
||||
func (gl *goLogger) Logf(level Level, format string, args ...interface{}) { |
||||
if gl.logger == nil { |
||||
return |
||||
} |
||||
if level <= gl.level { |
||||
e := newLogEntry(fmt.Sprintf(format, args...), level) |
||||
gl.log(e) |
||||
} |
||||
} |
||||
|
||||
func (gl *goLogger) Debugf(format string, args ...interface{}) { |
||||
gl.Logf(DEBUG, format, args...) |
||||
} |
||||
|
||||
func (gl *goLogger) Tracef(format string, args ...interface{}) { |
||||
gl.Logf(TRACE, format, args...) |
||||
} |
||||
|
||||
func (gl *goLogger) Requestf(rlevel Level, format string, args ...interface{}) { |
||||
if gl.logger == nil { |
||||
return |
||||
} |
||||
if REQUEST <= gl.level { |
||||
e := newLogEntry(fmt.Sprintf(format, args...), REQUEST) |
||||
e.Rlevel = rlevel |
||||
gl.log(e) |
||||
} |
||||
} |
||||
|
||||
func (gl *goLogger) Infof(format string, args ...interface{}) { |
||||
gl.Logf(INFO, format, args...) |
||||
} |
||||
|
||||
func (gl *goLogger) Warnf(format string, args ...interface{}) { |
||||
gl.Logf(WARN, format, args...) |
||||
} |
||||
|
||||
func (gl *goLogger) Errorf(format string, args ...interface{}) { |
||||
gl.Logf(ERROR, format, args...) |
||||
} |
||||
|
||||
func (gl *goLogger) Severef(format string, args ...interface{}) { |
||||
gl.Logf(SEVERE, format, args...) |
||||
} |
||||
|
||||
func (gl *goLogger) Fatalf(format string, args ...interface{}) { |
||||
gl.Logf(FATAL, format, args...) |
||||
} |
||||
|
||||
func (gl *goLogger) Level() Level { |
||||
return gl.level |
||||
} |
||||
|
||||
func (gl *goLogger) SetLevel(level Level) { |
||||
gl.level = level |
||||
} |
||||
|
||||
func (gl *goLogger) log(newEntry *logEntry) { |
||||
s := gl.entryFormatter.format(newEntry) |
||||
gl.logger.Print(s) |
||||
} |
||||
|
||||
type logEntry struct { |
||||
Time string |
||||
Level Level |
||||
Rlevel Level |
||||
Message string |
||||
Data Map |
||||
} |
||||
|
||||
func newLogEntry(msg string, level Level) *logEntry { |
||||
return &logEntry{ |
||||
Time: time.Now().Format("2006-01-02T15:04:05.000-07:00"), // time.RFC3339 with milliseconds
|
||||
Level: level, |
||||
Rlevel: NONE, |
||||
Message: msg, |
||||
} |
||||
} |
||||
|
||||
func copyPairs(newEntry *logEntry, pairs []Pair) { |
||||
newEntry.Data = make(Map, len(pairs)) |
||||
for _, p := range pairs { |
||||
newEntry.Data[p.Name] = p.Value |
||||
} |
||||
} |
||||
|
||||
type formatter interface { |
||||
format(*logEntry) string |
||||
} |
||||
|
||||
type textFormatter struct { |
||||
} |
||||
|
||||
// ex. 2016-02-10T09:15:25.498-08:00 [INFO] This is a message from test in text format
|
||||
|
||||
func (*textFormatter) format(newEntry *logEntry) string { |
||||
b := &bytes.Buffer{} |
||||
appendValue(b, newEntry.Time) |
||||
if newEntry.Rlevel != NONE { |
||||
fmt.Fprintf(b, "[%s,%s] ", newEntry.Level.String(), newEntry.Rlevel.String()) |
||||
} else { |
||||
fmt.Fprintf(b, "[%s] ", newEntry.Level.String()) |
||||
} |
||||
appendValue(b, newEntry.Message) |
||||
for key, value := range newEntry.Data { |
||||
appendKeyValue(b, key, value) |
||||
} |
||||
b.WriteByte('\n') |
||||
s := bytes.NewBuffer(b.Bytes()) |
||||
return s.String() |
||||
} |
||||
|
||||
func appendValue(b *bytes.Buffer, value interface{}) { |
||||
if _, ok := value.(string); ok { |
||||
fmt.Fprintf(b, "%s ", value) |
||||
} else { |
||||
fmt.Fprintf(b, "%v ", value) |
||||
} |
||||
} |
||||
|
||||
type keyvalueFormatter struct { |
||||
} |
||||
|
||||
// ex. _time=2016-02-10T09:15:25.498-08:00 _level=INFO _msg=This is a message from test in key-value format
|
||||
|
||||
func (*keyvalueFormatter) format(newEntry *logEntry) string { |
||||
b := &bytes.Buffer{} |
||||
appendKeyValue(b, _TIME, newEntry.Time) |
||||
appendKeyValue(b, _LEVEL, newEntry.Level.String()) |
||||
if newEntry.Rlevel != NONE { |
||||
appendKeyValue(b, _RLEVEL, newEntry.Rlevel.String()) |
||||
} |
||||
appendKeyValue(b, _MSG, newEntry.Message) |
||||
for key, value := range newEntry.Data { |
||||
appendKeyValue(b, key, value) |
||||
} |
||||
b.WriteByte('\n') |
||||
s := bytes.NewBuffer(b.Bytes()) |
||||
return s.String() |
||||
} |
||||
|
||||
func appendKeyValue(b *bytes.Buffer, key, value interface{}) { |
||||
if _, ok := value.(string); ok { |
||||
fmt.Fprintf(b, "%v=%s ", key, value) |
||||
} else { |
||||
fmt.Fprintf(b, "%v=%v ", key, value) |
||||
} |
||||
} |
||||
|
||||
type jsonFormatter struct { |
||||
} |
||||
|
||||
// ex. {"_level":"INFO","_msg":"This is a message from test in json format","_time":"2016-02-10T09:12:59.518-08:00"}
|
||||
|
||||
func (*jsonFormatter) format(newEntry *logEntry) string { |
||||
if newEntry.Data == nil { |
||||
newEntry.Data = make(Map, 5) |
||||
} |
||||
newEntry.Data[_TIME] = newEntry.Time |
||||
newEntry.Data[_LEVEL] = newEntry.Level.String() |
||||
if newEntry.Rlevel != NONE { |
||||
newEntry.Data[_RLEVEL] = newEntry.Rlevel.String() |
||||
} |
||||
newEntry.Data[_MSG] = newEntry.Message |
||||
serialized, _ := json.Marshal(newEntry.Data) |
||||
s := bytes.NewBuffer(append(serialized, '\n')) |
||||
return s.String() |
||||
} |
||||
@ -0,0 +1,207 @@ |
||||
// @author Couchbase <info@couchbase.com>
|
||||
// @copyright 2018 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package scramsha provides implementation of client side SCRAM-SHA
|
||||
// according to https://tools.ietf.org/html/rfc5802
|
||||
package scramsha |
||||
|
||||
import ( |
||||
"crypto/hmac" |
||||
"crypto/rand" |
||||
"crypto/sha1" |
||||
"crypto/sha256" |
||||
"crypto/sha512" |
||||
"encoding/base64" |
||||
"fmt" |
||||
"github.com/pkg/errors" |
||||
"golang.org/x/crypto/pbkdf2" |
||||
"hash" |
||||
"strconv" |
||||
"strings" |
||||
) |
||||
|
||||
func hmacHash(message []byte, secret []byte, hashFunc func() hash.Hash) []byte { |
||||
h := hmac.New(hashFunc, secret) |
||||
h.Write(message) |
||||
return h.Sum(nil) |
||||
} |
||||
|
||||
func shaHash(message []byte, hashFunc func() hash.Hash) []byte { |
||||
h := hashFunc() |
||||
h.Write(message) |
||||
return h.Sum(nil) |
||||
} |
||||
|
||||
func generateClientNonce(size int) (string, error) { |
||||
randomBytes := make([]byte, size) |
||||
_, err := rand.Read(randomBytes) |
||||
if err != nil { |
||||
return "", errors.Wrap(err, "Unable to generate nonce") |
||||
} |
||||
return base64.StdEncoding.EncodeToString(randomBytes), nil |
||||
} |
||||
|
||||
// ScramSha provides context for SCRAM-SHA handling
|
||||
type ScramSha struct { |
||||
hashSize int |
||||
hashFunc func() hash.Hash |
||||
clientNonce string |
||||
serverNonce string |
||||
salt []byte |
||||
i int |
||||
saltedPassword []byte |
||||
authMessage string |
||||
} |
||||
|
||||
var knownMethods = []string{"SCRAM-SHA512", "SCRAM-SHA256", "SCRAM-SHA1"} |
||||
|
||||
// BestMethod returns SCRAM-SHA method we consider the best out of suggested
|
||||
// by server
|
||||
func BestMethod(methods string) (string, error) { |
||||
for _, m := range knownMethods { |
||||
if strings.Index(methods, m) != -1 { |
||||
return m, nil |
||||
} |
||||
} |
||||
return "", errors.Errorf( |
||||
"None of the server suggested methods [%s] are supported", |
||||
methods) |
||||
} |
||||
|
||||
// NewScramSha creates context for SCRAM-SHA handling
|
||||
func NewScramSha(method string) (*ScramSha, error) { |
||||
s := &ScramSha{} |
||||
|
||||
if method == knownMethods[0] { |
||||
s.hashFunc = sha512.New |
||||
s.hashSize = 64 |
||||
} else if method == knownMethods[1] { |
||||
s.hashFunc = sha256.New |
||||
s.hashSize = 32 |
||||
} else if method == knownMethods[2] { |
||||
s.hashFunc = sha1.New |
||||
s.hashSize = 20 |
||||
} else { |
||||
return nil, errors.Errorf("Unsupported method %s", method) |
||||
} |
||||
return s, nil |
||||
} |
||||
|
||||
// GetStartRequest builds start SCRAM-SHA request to be sent to server
|
||||
func (s *ScramSha) GetStartRequest(user string) (string, error) { |
||||
var err error |
||||
s.clientNonce, err = generateClientNonce(24) |
||||
if err != nil { |
||||
return "", errors.Wrapf(err, "Unable to generate SCRAM-SHA "+ |
||||
"start request for user %s", user) |
||||
} |
||||
|
||||
message := fmt.Sprintf("n,,n=%s,r=%s", user, s.clientNonce) |
||||
s.authMessage = message[3:] |
||||
return message, nil |
||||
} |
||||
|
||||
// HandleStartResponse handles server response on start SCRAM-SHA request
|
||||
func (s *ScramSha) HandleStartResponse(response string) error { |
||||
parts := strings.Split(response, ",") |
||||
if len(parts) != 3 { |
||||
return errors.Errorf("expected 3 fields in first SCRAM-SHA-1 "+ |
||||
"server message %s", response) |
||||
} |
||||
if !strings.HasPrefix(parts[0], "r=") || len(parts[0]) < 3 { |
||||
return errors.Errorf("Server sent an invalid nonce %s", |
||||
parts[0]) |
||||
} |
||||
if !strings.HasPrefix(parts[1], "s=") || len(parts[1]) < 3 { |
||||
return errors.Errorf("Server sent an invalid salt %s", parts[1]) |
||||
} |
||||
if !strings.HasPrefix(parts[2], "i=") || len(parts[2]) < 3 { |
||||
return errors.Errorf("Server sent an invalid iteration count %s", |
||||
parts[2]) |
||||
} |
||||
|
||||
s.serverNonce = parts[0][2:] |
||||
encodedSalt := parts[1][2:] |
||||
var err error |
||||
s.i, err = strconv.Atoi(parts[2][2:]) |
||||
if err != nil { |
||||
return errors.Errorf("Iteration count %s must be integer.", |
||||
parts[2][2:]) |
||||
} |
||||
|
||||
if s.i < 1 { |
||||
return errors.New("Iteration count should be positive") |
||||
} |
||||
|
||||
if !strings.HasPrefix(s.serverNonce, s.clientNonce) { |
||||
return errors.Errorf("Server nonce %s doesn't contain client"+ |
||||
" nonce %s", s.serverNonce, s.clientNonce) |
||||
} |
||||
|
||||
s.salt, err = base64.StdEncoding.DecodeString(encodedSalt) |
||||
if err != nil { |
||||
return errors.Wrapf(err, "Unable to decode salt %s", |
||||
encodedSalt) |
||||
} |
||||
|
||||
s.authMessage = s.authMessage + "," + response |
||||
return nil |
||||
} |
||||
|
||||
// GetFinalRequest builds final SCRAM-SHA request to be sent to server
|
||||
func (s *ScramSha) GetFinalRequest(pass string) string { |
||||
clientFinalMessageBare := "c=biws,r=" + s.serverNonce |
||||
s.authMessage = s.authMessage + "," + clientFinalMessageBare |
||||
|
||||
s.saltedPassword = pbkdf2.Key([]byte(pass), s.salt, s.i, |
||||
s.hashSize, s.hashFunc) |
||||
|
||||
clientKey := hmacHash([]byte("Client Key"), s.saltedPassword, s.hashFunc) |
||||
storedKey := shaHash(clientKey, s.hashFunc) |
||||
clientSignature := hmacHash([]byte(s.authMessage), storedKey, s.hashFunc) |
||||
|
||||
clientProof := make([]byte, len(clientSignature)) |
||||
for i := 0; i < len(clientSignature); i++ { |
||||
clientProof[i] = clientKey[i] ^ clientSignature[i] |
||||
} |
||||
|
||||
return clientFinalMessageBare + ",p=" + |
||||
base64.StdEncoding.EncodeToString(clientProof) |
||||
} |
||||
|
||||
// HandleFinalResponse handles server's response on final SCRAM-SHA request
|
||||
func (s *ScramSha) HandleFinalResponse(response string) error { |
||||
if strings.Contains(response, ",") || |
||||
!strings.HasPrefix(response, "v=") { |
||||
return errors.Errorf("Server sent an invalid final message %s", |
||||
response) |
||||
} |
||||
|
||||
decodedMessage, err := base64.StdEncoding.DecodeString(response[2:]) |
||||
if err != nil { |
||||
return errors.Wrapf(err, "Unable to decode server message %s", |
||||
response[2:]) |
||||
} |
||||
serverKey := hmacHash([]byte("Server Key"), s.saltedPassword, |
||||
s.hashFunc) |
||||
serverSignature := hmacHash([]byte(s.authMessage), serverKey, |
||||
s.hashFunc) |
||||
if string(decodedMessage) != string(serverSignature) { |
||||
return errors.Errorf("Server proof %s doesn't match "+ |
||||
"the expected: %s", |
||||
string(decodedMessage), string(serverSignature)) |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,252 @@ |
||||
// @author Couchbase <info@couchbase.com>
|
||||
// @copyright 2018 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package scramsha provides implementation of client side SCRAM-SHA
|
||||
// via Http according to https://tools.ietf.org/html/rfc7804
|
||||
package scramsha |
||||
|
||||
import ( |
||||
"encoding/base64" |
||||
"github.com/pkg/errors" |
||||
"io" |
||||
"io/ioutil" |
||||
"net/http" |
||||
"strings" |
||||
) |
||||
|
||||
// consts used to parse scramsha response from target
|
||||
const ( |
||||
WWWAuthenticate = "WWW-Authenticate" |
||||
AuthenticationInfo = "Authentication-Info" |
||||
Authorization = "Authorization" |
||||
DataPrefix = "data=" |
||||
SidPrefix = "sid=" |
||||
) |
||||
|
||||
// Request provides implementation of http request that can be retried
|
||||
type Request struct { |
||||
body io.ReadSeeker |
||||
|
||||
// Embed an HTTP request directly. This makes a *Request act exactly
|
||||
// like an *http.Request so that all meta methods are supported.
|
||||
*http.Request |
||||
} |
||||
|
||||
type lenReader interface { |
||||
Len() int |
||||
} |
||||
|
||||
// NewRequest creates http request that can be retried
|
||||
func NewRequest(method, url string, body io.ReadSeeker) (*Request, error) { |
||||
// Wrap the body in a noop ReadCloser if non-nil. This prevents the
|
||||
// reader from being closed by the HTTP client.
|
||||
var rcBody io.ReadCloser |
||||
if body != nil { |
||||
rcBody = ioutil.NopCloser(body) |
||||
} |
||||
|
||||
// Make the request with the noop-closer for the body.
|
||||
httpReq, err := http.NewRequest(method, url, rcBody) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
// Check if we can set the Content-Length automatically.
|
||||
if lr, ok := body.(lenReader); ok { |
||||
httpReq.ContentLength = int64(lr.Len()) |
||||
} |
||||
|
||||
return &Request{body, httpReq}, nil |
||||
} |
||||
|
||||
func encode(str string) string { |
||||
return base64.StdEncoding.EncodeToString([]byte(str)) |
||||
} |
||||
|
||||
func decode(str string) (string, error) { |
||||
bytes, err := base64.StdEncoding.DecodeString(str) |
||||
if err != nil { |
||||
return "", errors.Errorf("Cannot base64 decode %s", |
||||
str) |
||||
} |
||||
return string(bytes), err |
||||
} |
||||
|
||||
func trimPrefix(s, prefix string) (string, error) { |
||||
l := len(s) |
||||
trimmed := strings.TrimPrefix(s, prefix) |
||||
if l == len(trimmed) { |
||||
return trimmed, errors.Errorf("Prefix %s not found in %s", |
||||
prefix, s) |
||||
} |
||||
return trimmed, nil |
||||
} |
||||
|
||||
func drainBody(resp *http.Response) { |
||||
defer resp.Body.Close() |
||||
io.Copy(ioutil.Discard, resp.Body) |
||||
} |
||||
|
||||
// DoScramSha performs SCRAM-SHA handshake via Http
|
||||
func DoScramSha(req *Request, |
||||
username string, |
||||
password string, |
||||
client *http.Client) (*http.Response, error) { |
||||
|
||||
method := "SCRAM-SHA-512" |
||||
s, err := NewScramSha("SCRAM-SHA512") |
||||
if err != nil { |
||||
return nil, errors.Wrap(err, |
||||
"Unable to initialize SCRAM-SHA handler") |
||||
} |
||||
|
||||
message, err := s.GetStartRequest(username) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
encodedMessage := method + " " + DataPrefix + encode(message) |
||||
|
||||
req.Header.Set(Authorization, encodedMessage) |
||||
|
||||
res, err := client.Do(req.Request) |
||||
if err != nil { |
||||
return nil, errors.Wrap(err, "Problem sending SCRAM-SHA start"+ |
||||
"request") |
||||
} |
||||
|
||||
if res.StatusCode != http.StatusUnauthorized { |
||||
return res, nil |
||||
} |
||||
|
||||
authHeader := res.Header.Get(WWWAuthenticate) |
||||
if authHeader == "" { |
||||
drainBody(res) |
||||
return nil, errors.Errorf("Header %s is not populated in "+ |
||||
"SCRAM-SHA start response", WWWAuthenticate) |
||||
} |
||||
|
||||
authHeader, err = trimPrefix(authHeader, method+" ") |
||||
if err != nil { |
||||
if strings.HasPrefix(authHeader, "Basic ") { |
||||
// user not found
|
||||
return res, nil |
||||
} |
||||
drainBody(res) |
||||
return nil, errors.Wrapf(err, "Error while parsing SCRAM-SHA "+ |
||||
"start response %s", authHeader) |
||||
} |
||||
|
||||
drainBody(res) |
||||
|
||||
sid, response, err := parseSidAndData(authHeader) |
||||
if err != nil { |
||||
return nil, errors.Wrapf(err, "Error while parsing SCRAM-SHA "+ |
||||
"start response %s", authHeader) |
||||
} |
||||
|
||||
err = s.HandleStartResponse(response) |
||||
if err != nil { |
||||
return nil, errors.Wrapf(err, "Error parsing SCRAM-SHA start "+ |
||||
"response %s", response) |
||||
} |
||||
|
||||
message = s.GetFinalRequest(password) |
||||
encodedMessage = method + " " + SidPrefix + sid + "," + DataPrefix + |
||||
encode(message) |
||||
|
||||
req.Header.Set(Authorization, encodedMessage) |
||||
|
||||
// rewind request body so it can be resent again
|
||||
if req.body != nil { |
||||
if _, err = req.body.Seek(0, 0); err != nil { |
||||
return nil, errors.Errorf("Failed to seek body: %v", |
||||
err) |
||||
} |
||||
} |
||||
|
||||
res, err = client.Do(req.Request) |
||||
if err != nil { |
||||
return nil, errors.Wrap(err, "Problem sending SCRAM-SHA final"+ |
||||
"request") |
||||
} |
||||
|
||||
if res.StatusCode == http.StatusUnauthorized { |
||||
// TODO retrieve and return error
|
||||
return res, nil |
||||
} |
||||
|
||||
if res.StatusCode >= http.StatusInternalServerError { |
||||
// in this case we cannot expect server to set headers properly
|
||||
return res, nil |
||||
} |
||||
|
||||
authHeader = res.Header.Get(AuthenticationInfo) |
||||
if authHeader == "" { |
||||
drainBody(res) |
||||
return nil, errors.Errorf("Header %s is not populated in "+ |
||||
"SCRAM-SHA final response", AuthenticationInfo) |
||||
} |
||||
|
||||
finalSid, response, err := parseSidAndData(authHeader) |
||||
if err != nil { |
||||
drainBody(res) |
||||
return nil, errors.Wrapf(err, "Error while parsing SCRAM-SHA "+ |
||||
"final response %s", authHeader) |
||||
} |
||||
|
||||
if finalSid != sid { |
||||
drainBody(res) |
||||
return nil, errors.Errorf("Sid %s returned by server "+ |
||||
"doesn't match the original sid %s", finalSid, sid) |
||||
} |
||||
|
||||
err = s.HandleFinalResponse(response) |
||||
if err != nil { |
||||
drainBody(res) |
||||
return nil, errors.Wrapf(err, |
||||
"Error handling SCRAM-SHA final server response %s", |
||||
response) |
||||
} |
||||
return res, nil |
||||
} |
||||
|
||||
func parseSidAndData(authHeader string) (string, string, error) { |
||||
sidIndex := strings.Index(authHeader, SidPrefix) |
||||
if sidIndex < 0 { |
||||
return "", "", errors.Errorf("Cannot find %s in %s", |
||||
SidPrefix, authHeader) |
||||
} |
||||
|
||||
sidEndIndex := strings.Index(authHeader, ",") |
||||
if sidEndIndex < 0 { |
||||
return "", "", errors.Errorf("Cannot find ',' in %s", |
||||
authHeader) |
||||
} |
||||
|
||||
sid := authHeader[sidIndex+len(SidPrefix) : sidEndIndex] |
||||
|
||||
dataIndex := strings.Index(authHeader, DataPrefix) |
||||
if dataIndex < 0 { |
||||
return "", "", errors.Errorf("Cannot find %s in %s", |
||||
DataPrefix, authHeader) |
||||
} |
||||
|
||||
data, err := decode(authHeader[dataIndex+len(DataPrefix):]) |
||||
if err != nil { |
||||
return "", "", err |
||||
} |
||||
return sid, data, nil |
||||
} |
||||
@ -0,0 +1,19 @@ |
||||
Copyright (c) 2013 Couchbase, Inc. |
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of |
||||
this software and associated documentation files (the "Software"), to deal in |
||||
the Software without restriction, including without limitation the rights to |
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies |
||||
of the Software, and to permit persons to whom the Software is furnished to do |
||||
so, subject to the following conditions: |
||||
|
||||
The above copyright notice and this permission notice shall be included in all |
||||
copies or substantial portions of the Software. |
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
||||
SOFTWARE. |
||||
@ -0,0 +1,32 @@ |
||||
package couchbase |
||||
|
||||
import () |
||||
|
||||
// Sample data:
|
||||
// {"disabled":["12333", "22244"],"uid":"132492431","auditdEnabled":true,
|
||||
// "disabledUsers":[{"name":"bill","domain":"local"},{"name":"bob","domain":"local"}],
|
||||
// "logPath":"/Users/johanlarson/Library/Application Support/Couchbase/var/lib/couchbase/logs",
|
||||
// "rotateInterval":86400,"rotateSize":20971520}
|
||||
type AuditSpec struct { |
||||
Disabled []uint32 `json:"disabled"` |
||||
Uid string `json:"uid"` |
||||
AuditdEnabled bool `json:"auditdEnabled` |
||||
DisabledUsers []AuditUser `json:"disabledUsers"` |
||||
LogPath string `json:"logPath"` |
||||
RotateInterval int64 `json:"rotateInterval"` |
||||
RotateSize int64 `json:"rotateSize"` |
||||
} |
||||
|
||||
type AuditUser struct { |
||||
Name string `json:"name"` |
||||
Domain string `json:"domain"` |
||||
} |
||||
|
||||
func (c *Client) GetAuditSpec() (*AuditSpec, error) { |
||||
ret := &AuditSpec{} |
||||
err := c.parseURLResponse("/settings/audit", ret) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return ret, nil |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,387 @@ |
||||
package couchbase |
||||
|
||||
import ( |
||||
"errors" |
||||
"sync/atomic" |
||||
"time" |
||||
|
||||
"github.com/couchbase/gomemcached" |
||||
"github.com/couchbase/gomemcached/client" |
||||
"github.com/couchbase/goutils/logging" |
||||
) |
||||
|
||||
// GenericMcdAuthHandler is a kind of AuthHandler that performs
|
||||
// special auth exchange (like non-standard auth, possibly followed by
|
||||
// select-bucket).
|
||||
type GenericMcdAuthHandler interface { |
||||
AuthHandler |
||||
AuthenticateMemcachedConn(host string, conn *memcached.Client) error |
||||
} |
||||
|
||||
// Error raised when a connection can't be retrieved from a pool.
|
||||
var TimeoutError = errors.New("timeout waiting to build connection") |
||||
var errClosedPool = errors.New("the connection pool is closed") |
||||
var errNoPool = errors.New("no connection pool") |
||||
|
||||
// Default timeout for retrieving a connection from the pool.
|
||||
var ConnPoolTimeout = time.Hour * 24 * 30 |
||||
|
||||
// overflow connection closer cycle time
|
||||
var ConnCloserInterval = time.Second * 30 |
||||
|
||||
// ConnPoolAvailWaitTime is the amount of time to wait for an existing
|
||||
// connection from the pool before considering the creation of a new
|
||||
// one.
|
||||
var ConnPoolAvailWaitTime = time.Millisecond |
||||
|
||||
type connectionPool struct { |
||||
host string |
||||
mkConn func(host string, ah AuthHandler) (*memcached.Client, error) |
||||
auth AuthHandler |
||||
connections chan *memcached.Client |
||||
createsem chan bool |
||||
bailOut chan bool |
||||
poolSize int |
||||
connCount uint64 |
||||
inUse bool |
||||
} |
||||
|
||||
func newConnectionPool(host string, ah AuthHandler, closer bool, poolSize, poolOverflow int) *connectionPool { |
||||
connSize := poolSize |
||||
if closer { |
||||
connSize += poolOverflow |
||||
} |
||||
rv := &connectionPool{ |
||||
host: host, |
||||
connections: make(chan *memcached.Client, connSize), |
||||
createsem: make(chan bool, poolSize+poolOverflow), |
||||
mkConn: defaultMkConn, |
||||
auth: ah, |
||||
poolSize: poolSize, |
||||
} |
||||
if closer { |
||||
rv.bailOut = make(chan bool, 1) |
||||
go rv.connCloser() |
||||
} |
||||
return rv |
||||
} |
||||
|
||||
// ConnPoolTimeout is notified whenever connections are acquired from a pool.
|
||||
var ConnPoolCallback func(host string, source string, start time.Time, err error) |
||||
|
||||
func defaultMkConn(host string, ah AuthHandler) (*memcached.Client, error) { |
||||
var features memcached.Features |
||||
|
||||
conn, err := memcached.Connect("tcp", host) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if TCPKeepalive == true { |
||||
conn.SetKeepAliveOptions(time.Duration(TCPKeepaliveInterval) * time.Second) |
||||
} |
||||
|
||||
if EnableMutationToken == true { |
||||
features = append(features, memcached.FeatureMutationToken) |
||||
} |
||||
if EnableDataType == true { |
||||
features = append(features, memcached.FeatureDataType) |
||||
} |
||||
|
||||
if EnableXattr == true { |
||||
features = append(features, memcached.FeatureXattr) |
||||
} |
||||
|
||||
if len(features) > 0 { |
||||
if DefaultTimeout > 0 { |
||||
conn.SetDeadline(getDeadline(noDeadline, DefaultTimeout)) |
||||
} |
||||
|
||||
res, err := conn.EnableFeatures(features) |
||||
|
||||
if DefaultTimeout > 0 { |
||||
conn.SetDeadline(noDeadline) |
||||
} |
||||
|
||||
if err != nil && isTimeoutError(err) { |
||||
conn.Close() |
||||
return nil, err |
||||
} |
||||
|
||||
if err != nil || res.Status != gomemcached.SUCCESS { |
||||
logging.Warnf("Unable to enable features %v", err) |
||||
} |
||||
} |
||||
|
||||
if gah, ok := ah.(GenericMcdAuthHandler); ok { |
||||
err = gah.AuthenticateMemcachedConn(host, conn) |
||||
if err != nil { |
||||
conn.Close() |
||||
return nil, err |
||||
} |
||||
return conn, nil |
||||
} |
||||
name, pass, bucket := ah.GetCredentials() |
||||
if name != "default" { |
||||
_, err = conn.Auth(name, pass) |
||||
if err != nil { |
||||
conn.Close() |
||||
return nil, err |
||||
} |
||||
// Select bucket (Required for cb_auth creds)
|
||||
// Required when doing auth with _admin credentials
|
||||
if bucket != "" && bucket != name { |
||||
_, err = conn.SelectBucket(bucket) |
||||
if err != nil { |
||||
conn.Close() |
||||
return nil, err |
||||
} |
||||
} |
||||
} |
||||
return conn, nil |
||||
} |
||||
|
||||
func (cp *connectionPool) Close() (err error) { |
||||
defer func() { |
||||
if recover() != nil { |
||||
err = errors.New("connectionPool.Close error") |
||||
} |
||||
}() |
||||
if cp.bailOut != nil { |
||||
|
||||
// defensively, we won't wait if the channel is full
|
||||
select { |
||||
case cp.bailOut <- false: |
||||
default: |
||||
} |
||||
} |
||||
close(cp.connections) |
||||
for c := range cp.connections { |
||||
c.Close() |
||||
} |
||||
return |
||||
} |
||||
|
||||
func (cp *connectionPool) Node() string { |
||||
return cp.host |
||||
} |
||||
|
||||
func (cp *connectionPool) GetWithTimeout(d time.Duration) (rv *memcached.Client, err error) { |
||||
if cp == nil { |
||||
return nil, errNoPool |
||||
} |
||||
|
||||
path := "" |
||||
|
||||
if ConnPoolCallback != nil { |
||||
defer func(path *string, start time.Time) { |
||||
ConnPoolCallback(cp.host, *path, start, err) |
||||
}(&path, time.Now()) |
||||
} |
||||
|
||||
path = "short-circuit" |
||||
|
||||
// short-circuit available connetions.
|
||||
select { |
||||
case rv, isopen := <-cp.connections: |
||||
if !isopen { |
||||
return nil, errClosedPool |
||||
} |
||||
atomic.AddUint64(&cp.connCount, 1) |
||||
return rv, nil |
||||
default: |
||||
} |
||||
|
||||
t := time.NewTimer(ConnPoolAvailWaitTime) |
||||
defer t.Stop() |
||||
|
||||
// Try to grab an available connection within 1ms
|
||||
select { |
||||
case rv, isopen := <-cp.connections: |
||||
path = "avail1" |
||||
if !isopen { |
||||
return nil, errClosedPool |
||||
} |
||||
atomic.AddUint64(&cp.connCount, 1) |
||||
return rv, nil |
||||
case <-t.C: |
||||
// No connection came around in time, let's see
|
||||
// whether we can get one or build a new one first.
|
||||
t.Reset(d) // Reuse the timer for the full timeout.
|
||||
select { |
||||
case rv, isopen := <-cp.connections: |
||||
path = "avail2" |
||||
if !isopen { |
||||
return nil, errClosedPool |
||||
} |
||||
atomic.AddUint64(&cp.connCount, 1) |
||||
return rv, nil |
||||
case cp.createsem <- true: |
||||
path = "create" |
||||
// Build a connection if we can't get a real one.
|
||||
// This can potentially be an overflow connection, or
|
||||
// a pooled connection.
|
||||
rv, err := cp.mkConn(cp.host, cp.auth) |
||||
if err != nil { |
||||
// On error, release our create hold
|
||||
<-cp.createsem |
||||
} else { |
||||
atomic.AddUint64(&cp.connCount, 1) |
||||
} |
||||
return rv, err |
||||
case <-t.C: |
||||
return nil, ErrTimeout |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (cp *connectionPool) Get() (*memcached.Client, error) { |
||||
return cp.GetWithTimeout(ConnPoolTimeout) |
||||
} |
||||
|
||||
func (cp *connectionPool) Return(c *memcached.Client) { |
||||
if c == nil { |
||||
return |
||||
} |
||||
|
||||
if cp == nil { |
||||
c.Close() |
||||
} |
||||
|
||||
if c.IsHealthy() { |
||||
defer func() { |
||||
if recover() != nil { |
||||
// This happens when the pool has already been
|
||||
// closed and we're trying to return a
|
||||
// connection to it anyway. Just close the
|
||||
// connection.
|
||||
c.Close() |
||||
} |
||||
}() |
||||
|
||||
select { |
||||
case cp.connections <- c: |
||||
default: |
||||
<-cp.createsem |
||||
c.Close() |
||||
} |
||||
} else { |
||||
<-cp.createsem |
||||
c.Close() |
||||
} |
||||
} |
||||
|
||||
// give the ability to discard a connection from a pool
|
||||
// useful for ditching connections to the wrong node after a rebalance
|
||||
func (cp *connectionPool) Discard(c *memcached.Client) { |
||||
<-cp.createsem |
||||
c.Close() |
||||
} |
||||
|
||||
// asynchronous connection closer
|
||||
func (cp *connectionPool) connCloser() { |
||||
var connCount uint64 |
||||
|
||||
t := time.NewTimer(ConnCloserInterval) |
||||
defer t.Stop() |
||||
|
||||
for { |
||||
connCount = cp.connCount |
||||
|
||||
// we don't exist anymore! bail out!
|
||||
select { |
||||
case <-cp.bailOut: |
||||
return |
||||
case <-t.C: |
||||
} |
||||
t.Reset(ConnCloserInterval) |
||||
|
||||
// no overflow connections open or sustained requests for connections
|
||||
// nothing to do until the next cycle
|
||||
if len(cp.connections) <= cp.poolSize || |
||||
ConnCloserInterval/ConnPoolAvailWaitTime < time.Duration(cp.connCount-connCount) { |
||||
continue |
||||
} |
||||
|
||||
// close overflow connections now that they are not needed
|
||||
for c := range cp.connections { |
||||
select { |
||||
case <-cp.bailOut: |
||||
return |
||||
default: |
||||
} |
||||
|
||||
// bail out if close did not work out
|
||||
if !cp.connCleanup(c) { |
||||
return |
||||
} |
||||
if len(cp.connections) <= cp.poolSize { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
// close connection with recovery on error
|
||||
func (cp *connectionPool) connCleanup(c *memcached.Client) (rv bool) { |
||||
|
||||
// just in case we are closing a connection after
|
||||
// bailOut has been sent but we haven't yet read it
|
||||
defer func() { |
||||
if recover() != nil { |
||||
rv = false |
||||
} |
||||
}() |
||||
rv = true |
||||
|
||||
c.Close() |
||||
<-cp.createsem |
||||
return |
||||
} |
||||
|
||||
func (cp *connectionPool) StartTapFeed(args *memcached.TapArguments) (*memcached.TapFeed, error) { |
||||
if cp == nil { |
||||
return nil, errNoPool |
||||
} |
||||
mc, err := cp.Get() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
// A connection can't be used after TAP; Dont' count it against the
|
||||
// connection pool capacity
|
||||
<-cp.createsem |
||||
|
||||
return mc.StartTapFeed(*args) |
||||
} |
||||
|
||||
const DEFAULT_WINDOW_SIZE = 20 * 1024 * 1024 // 20 Mb
|
||||
|
||||
func (cp *connectionPool) StartUprFeed(name string, sequence uint32, dcp_buffer_size uint32, data_chan_size int) (*memcached.UprFeed, error) { |
||||
if cp == nil { |
||||
return nil, errNoPool |
||||
} |
||||
mc, err := cp.Get() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
// A connection can't be used after it has been allocated to UPR;
|
||||
// Dont' count it against the connection pool capacity
|
||||
<-cp.createsem |
||||
|
||||
uf, err := mc.NewUprFeed() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if err := uf.UprOpen(name, sequence, dcp_buffer_size); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if err := uf.StartFeedWithConfig(data_chan_size); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return uf, nil |
||||
} |
||||
@ -0,0 +1,288 @@ |
||||
package couchbase |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/json" |
||||
"fmt" |
||||
"github.com/couchbase/goutils/logging" |
||||
"io/ioutil" |
||||
"net/http" |
||||
) |
||||
|
||||
// ViewDefinition represents a single view within a design document.
|
||||
type ViewDefinition struct { |
||||
Map string `json:"map"` |
||||
Reduce string `json:"reduce,omitempty"` |
||||
} |
||||
|
||||
// DDoc is the document body of a design document specifying a view.
|
||||
type DDoc struct { |
||||
Language string `json:"language,omitempty"` |
||||
Views map[string]ViewDefinition `json:"views"` |
||||
} |
||||
|
||||
// DDocsResult represents the result from listing the design
|
||||
// documents.
|
||||
type DDocsResult struct { |
||||
Rows []struct { |
||||
DDoc struct { |
||||
Meta map[string]interface{} |
||||
JSON DDoc |
||||
} `json:"doc"` |
||||
} `json:"rows"` |
||||
} |
||||
|
||||
// GetDDocs lists all design documents
|
||||
func (b *Bucket) GetDDocs() (DDocsResult, error) { |
||||
var ddocsResult DDocsResult |
||||
b.RLock() |
||||
pool := b.pool |
||||
uri := b.DDocs.URI |
||||
b.RUnlock() |
||||
|
||||
// MB-23555 ephemeral buckets have no ddocs
|
||||
if uri == "" { |
||||
return DDocsResult{}, nil |
||||
} |
||||
|
||||
err := pool.client.parseURLResponse(uri, &ddocsResult) |
||||
if err != nil { |
||||
return DDocsResult{}, err |
||||
} |
||||
return ddocsResult, nil |
||||
} |
||||
|
||||
func (b *Bucket) GetDDocWithRetry(docname string, into interface{}) error { |
||||
ddocURI := fmt.Sprintf("/%s/_design/%s", b.GetName(), docname) |
||||
err := b.parseAPIResponse(ddocURI, &into) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (b *Bucket) GetDDocsWithRetry() (DDocsResult, error) { |
||||
var ddocsResult DDocsResult |
||||
b.RLock() |
||||
uri := b.DDocs.URI |
||||
b.RUnlock() |
||||
|
||||
// MB-23555 ephemeral buckets have no ddocs
|
||||
if uri == "" { |
||||
return DDocsResult{}, nil |
||||
} |
||||
|
||||
err := b.parseURLResponse(uri, &ddocsResult) |
||||
if err != nil { |
||||
return DDocsResult{}, err |
||||
} |
||||
return ddocsResult, nil |
||||
} |
||||
|
||||
func (b *Bucket) ddocURL(docname string) (string, error) { |
||||
u, err := b.randomBaseURL() |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
u.Path = fmt.Sprintf("/%s/_design/%s", b.GetName(), docname) |
||||
return u.String(), nil |
||||
} |
||||
|
||||
func (b *Bucket) ddocURLNext(nodeId int, docname string) (string, int, error) { |
||||
u, selected, err := b.randomNextURL(nodeId) |
||||
if err != nil { |
||||
return "", -1, err |
||||
} |
||||
u.Path = fmt.Sprintf("/%s/_design/%s", b.GetName(), docname) |
||||
return u.String(), selected, nil |
||||
} |
||||
|
||||
const ABS_MAX_RETRIES = 10 |
||||
const ABS_MIN_RETRIES = 3 |
||||
|
||||
func (b *Bucket) getMaxRetries() (int, error) { |
||||
|
||||
maxRetries := len(b.Nodes()) |
||||
|
||||
if maxRetries == 0 { |
||||
return 0, fmt.Errorf("No available Couch rest URLs") |
||||
} |
||||
|
||||
if maxRetries > ABS_MAX_RETRIES { |
||||
maxRetries = ABS_MAX_RETRIES |
||||
} else if maxRetries < ABS_MIN_RETRIES { |
||||
maxRetries = ABS_MIN_RETRIES |
||||
} |
||||
|
||||
return maxRetries, nil |
||||
} |
||||
|
||||
// PutDDoc installs a design document.
|
||||
func (b *Bucket) PutDDoc(docname string, value interface{}) error { |
||||
|
||||
var Err error |
||||
|
||||
maxRetries, err := b.getMaxRetries() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
lastNode := START_NODE_ID |
||||
|
||||
for retryCount := 0; retryCount < maxRetries; retryCount++ { |
||||
|
||||
Err = nil |
||||
|
||||
ddocU, selectedNode, err := b.ddocURLNext(lastNode, docname) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
lastNode = selectedNode |
||||
|
||||
logging.Infof(" Trying with selected node %d", selectedNode) |
||||
j, err := json.Marshal(value) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
req, err := http.NewRequest("PUT", ddocU, bytes.NewReader(j)) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
req.Header.Set("Content-Type", "application/json") |
||||
err = maybeAddAuth(req, b.authHandler(false /* bucket not yet locked */)) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
res, err := doHTTPRequest(req) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if res.StatusCode != 201 { |
||||
body, _ := ioutil.ReadAll(res.Body) |
||||
Err = fmt.Errorf("error installing view: %v / %s", |
||||
res.Status, body) |
||||
logging.Errorf(" Error in PutDDOC %v. Retrying...", Err) |
||||
res.Body.Close() |
||||
b.Refresh() |
||||
continue |
||||
} |
||||
|
||||
res.Body.Close() |
||||
break |
||||
} |
||||
|
||||
return Err |
||||
} |
||||
|
||||
// GetDDoc retrieves a specific a design doc.
|
||||
func (b *Bucket) GetDDoc(docname string, into interface{}) error { |
||||
var Err error |
||||
var res *http.Response |
||||
|
||||
maxRetries, err := b.getMaxRetries() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
lastNode := START_NODE_ID |
||||
for retryCount := 0; retryCount < maxRetries; retryCount++ { |
||||
|
||||
Err = nil |
||||
ddocU, selectedNode, err := b.ddocURLNext(lastNode, docname) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
lastNode = selectedNode |
||||
logging.Infof(" Trying with selected node %d", selectedNode) |
||||
|
||||
req, err := http.NewRequest("GET", ddocU, nil) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
req.Header.Set("Content-Type", "application/json") |
||||
err = maybeAddAuth(req, b.authHandler(false /* bucket not yet locked */)) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
res, err = doHTTPRequest(req) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if res.StatusCode != 200 { |
||||
body, _ := ioutil.ReadAll(res.Body) |
||||
Err = fmt.Errorf("error reading view: %v / %s", |
||||
res.Status, body) |
||||
logging.Errorf(" Error in GetDDOC %v Retrying...", Err) |
||||
b.Refresh() |
||||
res.Body.Close() |
||||
continue |
||||
} |
||||
defer res.Body.Close() |
||||
break |
||||
} |
||||
|
||||
if Err != nil { |
||||
return Err |
||||
} |
||||
|
||||
d := json.NewDecoder(res.Body) |
||||
return d.Decode(into) |
||||
} |
||||
|
||||
// DeleteDDoc removes a design document.
|
||||
func (b *Bucket) DeleteDDoc(docname string) error { |
||||
|
||||
var Err error |
||||
|
||||
maxRetries, err := b.getMaxRetries() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
lastNode := START_NODE_ID |
||||
|
||||
for retryCount := 0; retryCount < maxRetries; retryCount++ { |
||||
|
||||
Err = nil |
||||
ddocU, selectedNode, err := b.ddocURLNext(lastNode, docname) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
lastNode = selectedNode |
||||
logging.Infof(" Trying with selected node %d", selectedNode) |
||||
|
||||
req, err := http.NewRequest("DELETE", ddocU, nil) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
req.Header.Set("Content-Type", "application/json") |
||||
err = maybeAddAuth(req, b.authHandler(false /* bucket not already locked */)) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
res, err := doHTTPRequest(req) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if res.StatusCode != 200 { |
||||
body, _ := ioutil.ReadAll(res.Body) |
||||
Err = fmt.Errorf("error deleting view : %v / %s", res.Status, body) |
||||
logging.Errorf(" Error in DeleteDDOC %v. Retrying ... ", Err) |
||||
b.Refresh() |
||||
res.Body.Close() |
||||
continue |
||||
} |
||||
|
||||
res.Body.Close() |
||||
break |
||||
} |
||||
return Err |
||||
} |
||||
@ -0,0 +1,300 @@ |
||||
package couchbase |
||||
|
||||
import ( |
||||
"fmt" |
||||
"github.com/couchbase/goutils/logging" |
||||
"sync" |
||||
) |
||||
|
||||
type PersistTo uint8 |
||||
|
||||
const ( |
||||
PersistNone = PersistTo(0x00) |
||||
PersistMaster = PersistTo(0x01) |
||||
PersistOne = PersistTo(0x02) |
||||
PersistTwo = PersistTo(0x03) |
||||
PersistThree = PersistTo(0x04) |
||||
PersistFour = PersistTo(0x05) |
||||
) |
||||
|
||||
type ObserveTo uint8 |
||||
|
||||
const ( |
||||
ObserveNone = ObserveTo(0x00) |
||||
ObserveReplicateOne = ObserveTo(0x01) |
||||
ObserveReplicateTwo = ObserveTo(0x02) |
||||
ObserveReplicateThree = ObserveTo(0x03) |
||||
ObserveReplicateFour = ObserveTo(0x04) |
||||
) |
||||
|
||||
type JobType uint8 |
||||
|
||||
const ( |
||||
OBSERVE = JobType(0x00) |
||||
PERSIST = JobType(0x01) |
||||
) |
||||
|
||||
type ObservePersistJob struct { |
||||
vb uint16 |
||||
vbuuid uint64 |
||||
hostname string |
||||
jobType JobType |
||||
failover uint8 |
||||
lastPersistedSeqNo uint64 |
||||
currentSeqNo uint64 |
||||
resultChan chan *ObservePersistJob |
||||
errorChan chan *OPErrResponse |
||||
} |
||||
|
||||
type OPErrResponse struct { |
||||
vb uint16 |
||||
vbuuid uint64 |
||||
err error |
||||
job *ObservePersistJob |
||||
} |
||||
|
||||
var ObservePersistPool = NewPool(1024) |
||||
var OPJobChan = make(chan *ObservePersistJob, 1024) |
||||
var OPJobDone = make(chan bool) |
||||
|
||||
var wg sync.WaitGroup |
||||
|
||||
func (b *Bucket) StartOPPollers(maxWorkers int) { |
||||
|
||||
for i := 0; i < maxWorkers; i++ { |
||||
go b.OPJobPoll() |
||||
wg.Add(1) |
||||
} |
||||
wg.Wait() |
||||
} |
||||
|
||||
func (b *Bucket) SetObserveAndPersist(nPersist PersistTo, nObserve ObserveTo) (err error) { |
||||
|
||||
numNodes := len(b.Nodes()) |
||||
if int(nPersist) > numNodes || int(nObserve) > numNodes { |
||||
return fmt.Errorf("Not enough healthy nodes in the cluster") |
||||
} |
||||
|
||||
if int(nPersist) > (b.Replicas+1) || int(nObserve) > b.Replicas { |
||||
return fmt.Errorf("Not enough replicas in the cluster") |
||||
} |
||||
|
||||
if EnableMutationToken == false { |
||||
return fmt.Errorf("Mutation Tokens not enabled ") |
||||
} |
||||
|
||||
b.ds = &DurablitySettings{Persist: PersistTo(nPersist), Observe: ObserveTo(nObserve)} |
||||
return |
||||
} |
||||
|
||||
func (b *Bucket) ObserveAndPersistPoll(vb uint16, vbuuid uint64, seqNo uint64) (err error, failover bool) { |
||||
b.RLock() |
||||
ds := b.ds |
||||
b.RUnlock() |
||||
|
||||
if ds == nil { |
||||
return |
||||
} |
||||
|
||||
nj := 0 // total number of jobs
|
||||
resultChan := make(chan *ObservePersistJob, 10) |
||||
errChan := make(chan *OPErrResponse, 10) |
||||
|
||||
nodes := b.GetNodeList(vb) |
||||
if int(ds.Observe) > len(nodes) || int(ds.Persist) > len(nodes) { |
||||
return fmt.Errorf("Not enough healthy nodes in the cluster"), false |
||||
} |
||||
|
||||
logging.Infof("Node list %v", nodes) |
||||
|
||||
if ds.Observe >= ObserveReplicateOne { |
||||
// create a job for each host
|
||||
for i := ObserveReplicateOne; i < ds.Observe+1; i++ { |
||||
opJob := ObservePersistPool.Get() |
||||
opJob.vb = vb |
||||
opJob.vbuuid = vbuuid |
||||
opJob.jobType = OBSERVE |
||||
opJob.hostname = nodes[i] |
||||
opJob.resultChan = resultChan |
||||
opJob.errorChan = errChan |
||||
|
||||
OPJobChan <- opJob |
||||
nj++ |
||||
|
||||
} |
||||
} |
||||
|
||||
if ds.Persist >= PersistMaster { |
||||
for i := PersistMaster; i < ds.Persist+1; i++ { |
||||
opJob := ObservePersistPool.Get() |
||||
opJob.vb = vb |
||||
opJob.vbuuid = vbuuid |
||||
opJob.jobType = PERSIST |
||||
opJob.hostname = nodes[i] |
||||
opJob.resultChan = resultChan |
||||
opJob.errorChan = errChan |
||||
|
||||
OPJobChan <- opJob |
||||
nj++ |
||||
|
||||
} |
||||
} |
||||
|
||||
ok := true |
||||
for ok { |
||||
select { |
||||
case res := <-resultChan: |
||||
jobDone := false |
||||
if res.failover == 0 { |
||||
// no failover
|
||||
if res.jobType == PERSIST { |
||||
if res.lastPersistedSeqNo >= seqNo { |
||||
jobDone = true |
||||
} |
||||
|
||||
} else { |
||||
if res.currentSeqNo >= seqNo { |
||||
jobDone = true |
||||
} |
||||
} |
||||
|
||||
if jobDone == true { |
||||
nj-- |
||||
ObservePersistPool.Put(res) |
||||
} else { |
||||
// requeue this job
|
||||
OPJobChan <- res |
||||
} |
||||
|
||||
} else { |
||||
// Not currently handling failover scenarios TODO
|
||||
nj-- |
||||
ObservePersistPool.Put(res) |
||||
failover = true |
||||
} |
||||
|
||||
if nj == 0 { |
||||
// done with all the jobs
|
||||
ok = false |
||||
close(resultChan) |
||||
close(errChan) |
||||
} |
||||
|
||||
case Err := <-errChan: |
||||
logging.Errorf("Error in Observe/Persist %v", Err.err) |
||||
err = fmt.Errorf("Error in Observe/Persist job %v", Err.err) |
||||
nj-- |
||||
ObservePersistPool.Put(Err.job) |
||||
if nj == 0 { |
||||
close(resultChan) |
||||
close(errChan) |
||||
ok = false |
||||
} |
||||
} |
||||
} |
||||
|
||||
return |
||||
} |
||||
|
||||
func (b *Bucket) OPJobPoll() { |
||||
|
||||
ok := true |
||||
for ok == true { |
||||
select { |
||||
case job := <-OPJobChan: |
||||
pool := b.getConnPoolByHost(job.hostname, false /* bucket not already locked */) |
||||
if pool == nil { |
||||
errRes := &OPErrResponse{vb: job.vb, vbuuid: job.vbuuid} |
||||
errRes.err = fmt.Errorf("Pool not found for host %v", job.hostname) |
||||
errRes.job = job |
||||
job.errorChan <- errRes |
||||
continue |
||||
} |
||||
conn, err := pool.Get() |
||||
if err != nil { |
||||
errRes := &OPErrResponse{vb: job.vb, vbuuid: job.vbuuid} |
||||
errRes.err = fmt.Errorf("Unable to get connection from pool %v", err) |
||||
errRes.job = job |
||||
job.errorChan <- errRes |
||||
continue |
||||
} |
||||
|
||||
res, err := conn.ObserveSeq(job.vb, job.vbuuid) |
||||
if err != nil { |
||||
errRes := &OPErrResponse{vb: job.vb, vbuuid: job.vbuuid} |
||||
errRes.err = fmt.Errorf("Command failed %v", err) |
||||
errRes.job = job |
||||
job.errorChan <- errRes |
||||
continue |
||||
|
||||
} |
||||
pool.Return(conn) |
||||
job.lastPersistedSeqNo = res.LastPersistedSeqNo |
||||
job.currentSeqNo = res.CurrentSeqNo |
||||
job.failover = res.Failover |
||||
|
||||
job.resultChan <- job |
||||
case <-OPJobDone: |
||||
logging.Infof("Observe Persist Poller exitting") |
||||
ok = false |
||||
} |
||||
} |
||||
wg.Done() |
||||
} |
||||
|
||||
func (b *Bucket) GetNodeList(vb uint16) []string { |
||||
|
||||
vbm := b.VBServerMap() |
||||
if len(vbm.VBucketMap) < int(vb) { |
||||
logging.Infof("vbmap smaller than vblist") |
||||
return nil |
||||
} |
||||
|
||||
nodes := make([]string, len(vbm.VBucketMap[vb])) |
||||
for i := 0; i < len(vbm.VBucketMap[vb]); i++ { |
||||
n := vbm.VBucketMap[vb][i] |
||||
if n < 0 { |
||||
continue |
||||
} |
||||
|
||||
node := b.getMasterNode(n) |
||||
if len(node) > 1 { |
||||
nodes[i] = node |
||||
} |
||||
continue |
||||
|
||||
} |
||||
return nodes |
||||
} |
||||
|
||||
//pool of ObservePersist Jobs
|
||||
type OPpool struct { |
||||
pool chan *ObservePersistJob |
||||
} |
||||
|
||||
// NewPool creates a new pool of jobs
|
||||
func NewPool(max int) *OPpool { |
||||
return &OPpool{ |
||||
pool: make(chan *ObservePersistJob, max), |
||||
} |
||||
} |
||||
|
||||
// Borrow a Client from the pool.
|
||||
func (p *OPpool) Get() *ObservePersistJob { |
||||
var o *ObservePersistJob |
||||
select { |
||||
case o = <-p.pool: |
||||
default: |
||||
o = &ObservePersistJob{} |
||||
} |
||||
return o |
||||
} |
||||
|
||||
// Return returns a Client to the pool.
|
||||
func (p *OPpool) Put(o *ObservePersistJob) { |
||||
select { |
||||
case p.pool <- o: |
||||
default: |
||||
// let it go, let it go...
|
||||
} |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,209 @@ |
||||
package couchbase |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"fmt" |
||||
"github.com/couchbase/goutils/logging" |
||||
"io" |
||||
"io/ioutil" |
||||
"math/rand" |
||||
"net" |
||||
"net/http" |
||||
"time" |
||||
"unsafe" |
||||
) |
||||
|
||||
// Bucket auto-updater gets the latest version of the bucket config from
|
||||
// the server. If the configuration has changed then updated the local
|
||||
// bucket information. If the bucket has been deleted then notify anyone
|
||||
// who is holding a reference to this bucket
|
||||
|
||||
const MAX_RETRY_COUNT = 5 |
||||
const DISCONNECT_PERIOD = 120 * time.Second |
||||
|
||||
type NotifyFn func(bucket string, err error) |
||||
|
||||
// Use TCP keepalive to detect half close sockets
|
||||
var updaterTransport http.RoundTripper = &http.Transport{ |
||||
Proxy: http.ProxyFromEnvironment, |
||||
Dial: (&net.Dialer{ |
||||
Timeout: 30 * time.Second, |
||||
KeepAlive: 30 * time.Second, |
||||
}).Dial, |
||||
} |
||||
|
||||
var updaterHTTPClient = &http.Client{Transport: updaterTransport} |
||||
|
||||
func doHTTPRequestForUpdate(req *http.Request) (*http.Response, error) { |
||||
|
||||
var err error |
||||
var res *http.Response |
||||
|
||||
for i := 0; i < HTTP_MAX_RETRY; i++ { |
||||
res, err = updaterHTTPClient.Do(req) |
||||
if err != nil && isHttpConnError(err) { |
||||
continue |
||||
} |
||||
break |
||||
} |
||||
|
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return res, err |
||||
} |
||||
|
||||
func (b *Bucket) RunBucketUpdater(notify NotifyFn) { |
||||
go func() { |
||||
err := b.UpdateBucket() |
||||
if err != nil { |
||||
if notify != nil { |
||||
notify(b.GetName(), err) |
||||
} |
||||
logging.Errorf(" Bucket Updater exited with err %v", err) |
||||
} |
||||
}() |
||||
} |
||||
|
||||
func (b *Bucket) replaceConnPools2(with []*connectionPool, bucketLocked bool) { |
||||
if !bucketLocked { |
||||
b.Lock() |
||||
defer b.Unlock() |
||||
} |
||||
old := b.connPools |
||||
b.connPools = unsafe.Pointer(&with) |
||||
if old != nil { |
||||
for _, pool := range *(*[]*connectionPool)(old) { |
||||
if pool != nil && pool.inUse == false { |
||||
pool.Close() |
||||
} |
||||
} |
||||
} |
||||
return |
||||
} |
||||
|
||||
func (b *Bucket) UpdateBucket() error { |
||||
|
||||
var failures int |
||||
var returnErr error |
||||
|
||||
for { |
||||
|
||||
if failures == MAX_RETRY_COUNT { |
||||
logging.Errorf(" Maximum failures reached. Exiting loop...") |
||||
return fmt.Errorf("Max failures reached. Last Error %v", returnErr) |
||||
} |
||||
|
||||
nodes := b.Nodes() |
||||
if len(nodes) < 1 { |
||||
return fmt.Errorf("No healthy nodes found") |
||||
} |
||||
|
||||
startNode := rand.Intn(len(nodes)) |
||||
node := nodes[(startNode)%len(nodes)] |
||||
|
||||
streamUrl := fmt.Sprintf("http://%s/pools/default/bucketsStreaming/%s", node.Hostname, b.GetName()) |
||||
logging.Infof(" Trying with %s", streamUrl) |
||||
req, err := http.NewRequest("GET", streamUrl, nil) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
b.RLock() |
||||
pool := b.pool |
||||
bucketName := b.Name |
||||
b.RUnlock() |
||||
scopes, err := getScopesAndCollections(pool, bucketName) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
// Lock here to avoid having pool closed under us.
|
||||
b.RLock() |
||||
err = maybeAddAuth(req, b.pool.client.ah) |
||||
b.RUnlock() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
res, err := doHTTPRequestForUpdate(req) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if res.StatusCode != 200 { |
||||
bod, _ := ioutil.ReadAll(io.LimitReader(res.Body, 512)) |
||||
logging.Errorf("Failed to connect to host, unexpected status code: %v. Body %s", res.StatusCode, bod) |
||||
res.Body.Close() |
||||
returnErr = fmt.Errorf("Failed to connect to host. Status %v Body %s", res.StatusCode, bod) |
||||
failures++ |
||||
continue |
||||
} |
||||
|
||||
dec := json.NewDecoder(res.Body) |
||||
|
||||
tmpb := &Bucket{} |
||||
for { |
||||
|
||||
err := dec.Decode(&tmpb) |
||||
if err != nil { |
||||
returnErr = err |
||||
res.Body.Close() |
||||
break |
||||
} |
||||
|
||||
// if we got here, reset failure count
|
||||
failures = 0 |
||||
b.Lock() |
||||
|
||||
// mark all the old connection pools for deletion
|
||||
pools := b.getConnPools(true /* already locked */) |
||||
for _, pool := range pools { |
||||
if pool != nil { |
||||
pool.inUse = false |
||||
} |
||||
} |
||||
|
||||
newcps := make([]*connectionPool, len(tmpb.VBSMJson.ServerList)) |
||||
for i := range newcps { |
||||
// get the old connection pool and check if it is still valid
|
||||
pool := b.getConnPoolByHost(tmpb.VBSMJson.ServerList[i], true /* bucket already locked */) |
||||
if pool != nil && pool.inUse == false { |
||||
// if the hostname and index is unchanged then reuse this pool
|
||||
newcps[i] = pool |
||||
pool.inUse = true |
||||
continue |
||||
} |
||||
// else create a new pool
|
||||
if b.ah != nil { |
||||
newcps[i] = newConnectionPool( |
||||
tmpb.VBSMJson.ServerList[i], |
||||
b.ah, false, PoolSize, PoolOverflow) |
||||
|
||||
} else { |
||||
newcps[i] = newConnectionPool( |
||||
tmpb.VBSMJson.ServerList[i], |
||||
b.authHandler(true /* bucket already locked */), |
||||
false, PoolSize, PoolOverflow) |
||||
} |
||||
} |
||||
|
||||
b.replaceConnPools2(newcps, true /* bucket already locked */) |
||||
|
||||
tmpb.ah = b.ah |
||||
b.vBucketServerMap = unsafe.Pointer(&tmpb.VBSMJson) |
||||
b.nodeList = unsafe.Pointer(&tmpb.NodesJSON) |
||||
b.Scopes = scopes |
||||
b.Unlock() |
||||
|
||||
logging.Infof("Got new configuration for bucket %s", b.GetName()) |
||||
|
||||
} |
||||
// we are here because of an error
|
||||
failures++ |
||||
continue |
||||
|
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,143 @@ |
||||
package couchbase |
||||
|
||||
import ( |
||||
"github.com/couchbase/gomemcached/client" |
||||
"github.com/couchbase/goutils/logging" |
||||
"sync" |
||||
"time" |
||||
) |
||||
|
||||
const initialRetryInterval = 1 * time.Second |
||||
const maximumRetryInterval = 30 * time.Second |
||||
|
||||
// A TapFeed streams mutation events from a bucket.
|
||||
//
|
||||
// Events from the bucket can be read from the channel 'C'. Remember
|
||||
// to call Close() on it when you're done, unless its channel has
|
||||
// closed itself already.
|
||||
type TapFeed struct { |
||||
C <-chan memcached.TapEvent |
||||
|
||||
bucket *Bucket |
||||
args *memcached.TapArguments |
||||
nodeFeeds []*memcached.TapFeed // The TAP feeds of the individual nodes
|
||||
output chan memcached.TapEvent // Same as C but writeably-typed
|
||||
wg sync.WaitGroup |
||||
quit chan bool |
||||
} |
||||
|
||||
// StartTapFeed creates and starts a new Tap feed
|
||||
func (b *Bucket) StartTapFeed(args *memcached.TapArguments) (*TapFeed, error) { |
||||
if args == nil { |
||||
defaultArgs := memcached.DefaultTapArguments() |
||||
args = &defaultArgs |
||||
} |
||||
|
||||
feed := &TapFeed{ |
||||
bucket: b, |
||||
args: args, |
||||
output: make(chan memcached.TapEvent, 10), |
||||
quit: make(chan bool), |
||||
} |
||||
|
||||
go feed.run() |
||||
|
||||
feed.C = feed.output |
||||
return feed, nil |
||||
} |
||||
|
||||
// Goroutine that runs the feed
|
||||
func (feed *TapFeed) run() { |
||||
retryInterval := initialRetryInterval |
||||
bucketOK := true |
||||
for { |
||||
// Connect to the TAP feed of each server node:
|
||||
if bucketOK { |
||||
killSwitch, err := feed.connectToNodes() |
||||
if err == nil { |
||||
// Run until one of the sub-feeds fails:
|
||||
select { |
||||
case <-killSwitch: |
||||
case <-feed.quit: |
||||
return |
||||
} |
||||
feed.closeNodeFeeds() |
||||
retryInterval = initialRetryInterval |
||||
} |
||||
} |
||||
|
||||
// On error, try to refresh the bucket in case the list of nodes changed:
|
||||
logging.Infof("go-couchbase: TAP connection lost; reconnecting to bucket %q in %v", |
||||
feed.bucket.Name, retryInterval) |
||||
err := feed.bucket.Refresh() |
||||
bucketOK = err == nil |
||||
|
||||
select { |
||||
case <-time.After(retryInterval): |
||||
case <-feed.quit: |
||||
return |
||||
} |
||||
if retryInterval *= 2; retryInterval > maximumRetryInterval { |
||||
retryInterval = maximumRetryInterval |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (feed *TapFeed) connectToNodes() (killSwitch chan bool, err error) { |
||||
killSwitch = make(chan bool) |
||||
for _, serverConn := range feed.bucket.getConnPools(false /* not already locked */) { |
||||
var singleFeed *memcached.TapFeed |
||||
singleFeed, err = serverConn.StartTapFeed(feed.args) |
||||
if err != nil { |
||||
logging.Errorf("go-couchbase: Error connecting to tap feed of %s: %v", serverConn.host, err) |
||||
feed.closeNodeFeeds() |
||||
return |
||||
} |
||||
feed.nodeFeeds = append(feed.nodeFeeds, singleFeed) |
||||
go feed.forwardTapEvents(singleFeed, killSwitch, serverConn.host) |
||||
feed.wg.Add(1) |
||||
} |
||||
return |
||||
} |
||||
|
||||
// Goroutine that forwards Tap events from a single node's feed to the aggregate feed.
|
||||
func (feed *TapFeed) forwardTapEvents(singleFeed *memcached.TapFeed, killSwitch chan bool, host string) { |
||||
defer feed.wg.Done() |
||||
for { |
||||
select { |
||||
case event, ok := <-singleFeed.C: |
||||
if !ok { |
||||
if singleFeed.Error != nil { |
||||
logging.Errorf("go-couchbase: Tap feed from %s failed: %v", host, singleFeed.Error) |
||||
} |
||||
killSwitch <- true |
||||
return |
||||
} |
||||
feed.output <- event |
||||
case <-feed.quit: |
||||
return |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (feed *TapFeed) closeNodeFeeds() { |
||||
for _, f := range feed.nodeFeeds { |
||||
f.Close() |
||||
} |
||||
feed.nodeFeeds = nil |
||||
} |
||||
|
||||
// Close a Tap feed.
|
||||
func (feed *TapFeed) Close() error { |
||||
select { |
||||
case <-feed.quit: |
||||
return nil |
||||
default: |
||||
} |
||||
|
||||
feed.closeNodeFeeds() |
||||
close(feed.quit) |
||||
feed.wg.Wait() |
||||
close(feed.output) |
||||
return nil |
||||
} |
||||
@ -0,0 +1,398 @@ |
||||
package couchbase |
||||
|
||||
import ( |
||||
"log" |
||||
"sync" |
||||
"time" |
||||
|
||||
"fmt" |
||||
"github.com/couchbase/gomemcached" |
||||
"github.com/couchbase/gomemcached/client" |
||||
"github.com/couchbase/goutils/logging" |
||||
) |
||||
|
||||
// A UprFeed streams mutation events from a bucket.
|
||||
//
|
||||
// Events from the bucket can be read from the channel 'C'. Remember
|
||||
// to call Close() on it when you're done, unless its channel has
|
||||
// closed itself already.
|
||||
type UprFeed struct { |
||||
C <-chan *memcached.UprEvent |
||||
|
||||
bucket *Bucket |
||||
nodeFeeds map[string]*FeedInfo // The UPR feeds of the individual nodes
|
||||
output chan *memcached.UprEvent // Same as C but writeably-typed
|
||||
outputClosed bool |
||||
quit chan bool |
||||
name string // name of this UPR feed
|
||||
sequence uint32 // sequence number for this feed
|
||||
connected bool |
||||
killSwitch chan bool |
||||
closing bool |
||||
wg sync.WaitGroup |
||||
dcp_buffer_size uint32 |
||||
data_chan_size int |
||||
} |
||||
|
||||
// UprFeed from a single connection
|
||||
type FeedInfo struct { |
||||
uprFeed *memcached.UprFeed // UPR feed handle
|
||||
host string // hostname
|
||||
connected bool // connected
|
||||
quit chan bool // quit channel
|
||||
} |
||||
|
||||
type FailoverLog map[uint16]memcached.FailoverLog |
||||
|
||||
// GetFailoverLogs, get the failover logs for a set of vbucket ids
|
||||
func (b *Bucket) GetFailoverLogs(vBuckets []uint16) (FailoverLog, error) { |
||||
|
||||
// map vbids to their corresponding hosts
|
||||
vbHostList := make(map[string][]uint16) |
||||
vbm := b.VBServerMap() |
||||
if len(vbm.VBucketMap) < len(vBuckets) { |
||||
return nil, fmt.Errorf("vbmap smaller than vbucket list: %v vs. %v", |
||||
vbm.VBucketMap, vBuckets) |
||||
} |
||||
|
||||
for _, vb := range vBuckets { |
||||
masterID := vbm.VBucketMap[vb][0] |
||||
master := b.getMasterNode(masterID) |
||||
if master == "" { |
||||
return nil, fmt.Errorf("No master found for vb %d", vb) |
||||
} |
||||
|
||||
vbList := vbHostList[master] |
||||
if vbList == nil { |
||||
vbList = make([]uint16, 0) |
||||
} |
||||
vbList = append(vbList, vb) |
||||
vbHostList[master] = vbList |
||||
} |
||||
|
||||
failoverLogMap := make(FailoverLog) |
||||
for _, serverConn := range b.getConnPools(false /* not already locked */) { |
||||
|
||||
vbList := vbHostList[serverConn.host] |
||||
if vbList == nil { |
||||
continue |
||||
} |
||||
|
||||
mc, err := serverConn.Get() |
||||
if err != nil { |
||||
logging.Infof("No Free connections for vblist %v", vbList) |
||||
return nil, fmt.Errorf("No Free connections for host %s", |
||||
serverConn.host) |
||||
|
||||
} |
||||
// close the connection so that it doesn't get reused for upr data
|
||||
// connection
|
||||
defer mc.Close() |
||||
failoverlogs, err := mc.UprGetFailoverLog(vbList) |
||||
if err != nil { |
||||
return nil, fmt.Errorf("Error getting failover log %s host %s", |
||||
err.Error(), serverConn.host) |
||||
|
||||
} |
||||
|
||||
for vb, log := range failoverlogs { |
||||
failoverLogMap[vb] = *log |
||||
} |
||||
} |
||||
|
||||
return failoverLogMap, nil |
||||
} |
||||
|
||||
func (b *Bucket) StartUprFeed(name string, sequence uint32) (*UprFeed, error) { |
||||
return b.StartUprFeedWithConfig(name, sequence, 10, DEFAULT_WINDOW_SIZE) |
||||
} |
||||
|
||||
// StartUprFeed creates and starts a new Upr feed
|
||||
// No data will be sent on the channel unless vbuckets streams are requested
|
||||
func (b *Bucket) StartUprFeedWithConfig(name string, sequence uint32, data_chan_size int, dcp_buffer_size uint32) (*UprFeed, error) { |
||||
|
||||
feed := &UprFeed{ |
||||
bucket: b, |
||||
output: make(chan *memcached.UprEvent, data_chan_size), |
||||
quit: make(chan bool), |
||||
nodeFeeds: make(map[string]*FeedInfo, 0), |
||||
name: name, |
||||
sequence: sequence, |
||||
killSwitch: make(chan bool), |
||||
dcp_buffer_size: dcp_buffer_size, |
||||
data_chan_size: data_chan_size, |
||||
} |
||||
|
||||
err := feed.connectToNodes() |
||||
if err != nil { |
||||
return nil, fmt.Errorf("Cannot connect to bucket %s", err.Error()) |
||||
} |
||||
feed.connected = true |
||||
go feed.run() |
||||
|
||||
feed.C = feed.output |
||||
return feed, nil |
||||
} |
||||
|
||||
// UprRequestStream starts a stream for a vb on a feed
|
||||
func (feed *UprFeed) UprRequestStream(vb uint16, opaque uint16, flags uint32, |
||||
vuuid, startSequence, endSequence, snapStart, snapEnd uint64) error { |
||||
|
||||
defer func() { |
||||
if r := recover(); r != nil { |
||||
log.Panicf("Panic in UprRequestStream. Feed %v Bucket %v", feed, feed.bucket) |
||||
} |
||||
}() |
||||
|
||||
vbm := feed.bucket.VBServerMap() |
||||
if len(vbm.VBucketMap) < int(vb) { |
||||
return fmt.Errorf("vbmap smaller than vbucket list: %v vs. %v", |
||||
vb, vbm.VBucketMap) |
||||
} |
||||
|
||||
if int(vb) >= len(vbm.VBucketMap) { |
||||
return fmt.Errorf("Invalid vbucket id %d", vb) |
||||
} |
||||
|
||||
masterID := vbm.VBucketMap[vb][0] |
||||
master := feed.bucket.getMasterNode(masterID) |
||||
if master == "" { |
||||
return fmt.Errorf("Master node not found for vbucket %d", vb) |
||||
} |
||||
singleFeed := feed.nodeFeeds[master] |
||||
if singleFeed == nil { |
||||
return fmt.Errorf("UprFeed for this host not found") |
||||
} |
||||
|
||||
if err := singleFeed.uprFeed.UprRequestStream(vb, opaque, flags, |
||||
vuuid, startSequence, endSequence, snapStart, snapEnd); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// UprCloseStream ends a vbucket stream.
|
||||
func (feed *UprFeed) UprCloseStream(vb, opaqueMSB uint16) error { |
||||
|
||||
defer func() { |
||||
if r := recover(); r != nil { |
||||
log.Panicf("Panic in UprCloseStream. Feed %v Bucket %v ", feed, feed.bucket) |
||||
} |
||||
}() |
||||
|
||||
vbm := feed.bucket.VBServerMap() |
||||
if len(vbm.VBucketMap) < int(vb) { |
||||
return fmt.Errorf("vbmap smaller than vbucket list: %v vs. %v", |
||||
vb, vbm.VBucketMap) |
||||
} |
||||
|
||||
if int(vb) >= len(vbm.VBucketMap) { |
||||
return fmt.Errorf("Invalid vbucket id %d", vb) |
||||
} |
||||
|
||||
masterID := vbm.VBucketMap[vb][0] |
||||
master := feed.bucket.getMasterNode(masterID) |
||||
if master == "" { |
||||
return fmt.Errorf("Master node not found for vbucket %d", vb) |
||||
} |
||||
singleFeed := feed.nodeFeeds[master] |
||||
if singleFeed == nil { |
||||
return fmt.Errorf("UprFeed for this host not found") |
||||
} |
||||
|
||||
if err := singleFeed.uprFeed.CloseStream(vb, opaqueMSB); err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// Goroutine that runs the feed
|
||||
func (feed *UprFeed) run() { |
||||
retryInterval := initialRetryInterval |
||||
bucketOK := true |
||||
for { |
||||
// Connect to the UPR feed of each server node:
|
||||
if bucketOK { |
||||
// Run until one of the sub-feeds fails:
|
||||
select { |
||||
case <-feed.killSwitch: |
||||
case <-feed.quit: |
||||
return |
||||
} |
||||
//feed.closeNodeFeeds()
|
||||
retryInterval = initialRetryInterval |
||||
} |
||||
|
||||
if feed.closing == true { |
||||
// we have been asked to shut down
|
||||
return |
||||
} |
||||
|
||||
// On error, try to refresh the bucket in case the list of nodes changed:
|
||||
logging.Infof("go-couchbase: UPR connection lost; reconnecting to bucket %q in %v", |
||||
feed.bucket.Name, retryInterval) |
||||
|
||||
if err := feed.bucket.Refresh(); err != nil { |
||||
// if we fail to refresh the bucket, exit the feed
|
||||
// MB-14917
|
||||
logging.Infof("Unable to refresh bucket %s ", err.Error()) |
||||
close(feed.output) |
||||
feed.outputClosed = true |
||||
feed.closeNodeFeeds() |
||||
return |
||||
} |
||||
|
||||
// this will only connect to nodes that are not connected or changed
|
||||
// user will have to reconnect the stream
|
||||
err := feed.connectToNodes() |
||||
if err != nil { |
||||
logging.Infof("Unable to connect to nodes..exit ") |
||||
close(feed.output) |
||||
feed.outputClosed = true |
||||
feed.closeNodeFeeds() |
||||
return |
||||
} |
||||
bucketOK = err == nil |
||||
|
||||
select { |
||||
case <-time.After(retryInterval): |
||||
case <-feed.quit: |
||||
return |
||||
} |
||||
if retryInterval *= 2; retryInterval > maximumRetryInterval { |
||||
retryInterval = maximumRetryInterval |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (feed *UprFeed) connectToNodes() (err error) { |
||||
nodeCount := 0 |
||||
for _, serverConn := range feed.bucket.getConnPools(false /* not already locked */) { |
||||
|
||||
// this maybe a reconnection, so check if the connection to the node
|
||||
// already exists. Connect only if the node is not found in the list
|
||||
// or connected == false
|
||||
nodeFeed := feed.nodeFeeds[serverConn.host] |
||||
|
||||
if nodeFeed != nil && nodeFeed.connected == true { |
||||
continue |
||||
} |
||||
|
||||
var singleFeed *memcached.UprFeed |
||||
var name string |
||||
if feed.name == "" { |
||||
name = "DefaultUprClient" |
||||
} else { |
||||
name = feed.name |
||||
} |
||||
singleFeed, err = serverConn.StartUprFeed(name, feed.sequence, feed.dcp_buffer_size, feed.data_chan_size) |
||||
if err != nil { |
||||
logging.Errorf("go-couchbase: Error connecting to upr feed of %s: %v", serverConn.host, err) |
||||
feed.closeNodeFeeds() |
||||
return |
||||
} |
||||
// add the node to the connection map
|
||||
feedInfo := &FeedInfo{ |
||||
uprFeed: singleFeed, |
||||
connected: true, |
||||
host: serverConn.host, |
||||
quit: make(chan bool), |
||||
} |
||||
feed.nodeFeeds[serverConn.host] = feedInfo |
||||
go feed.forwardUprEvents(feedInfo, feed.killSwitch, serverConn.host) |
||||
feed.wg.Add(1) |
||||
nodeCount++ |
||||
} |
||||
if nodeCount == 0 { |
||||
return fmt.Errorf("No connection to bucket") |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// Goroutine that forwards Upr events from a single node's feed to the aggregate feed.
|
||||
func (feed *UprFeed) forwardUprEvents(nodeFeed *FeedInfo, killSwitch chan bool, host string) { |
||||
singleFeed := nodeFeed.uprFeed |
||||
|
||||
defer func() { |
||||
feed.wg.Done() |
||||
if r := recover(); r != nil { |
||||
//if feed is not closing, re-throw the panic
|
||||
if feed.outputClosed != true && feed.closing != true { |
||||
panic(r) |
||||
} else { |
||||
logging.Errorf("Panic is recovered. Since feed is closed, exit gracefully") |
||||
|
||||
} |
||||
} |
||||
}() |
||||
|
||||
for { |
||||
select { |
||||
case <-nodeFeed.quit: |
||||
nodeFeed.connected = false |
||||
return |
||||
|
||||
case event, ok := <-singleFeed.C: |
||||
if !ok { |
||||
if singleFeed.Error != nil { |
||||
logging.Errorf("go-couchbase: Upr feed from %s failed: %v", host, singleFeed.Error) |
||||
} |
||||
killSwitch <- true |
||||
return |
||||
} |
||||
if feed.outputClosed == true { |
||||
// someone closed the node feed
|
||||
logging.Infof("Node need closed, returning from forwardUprEvent") |
||||
return |
||||
} |
||||
feed.output <- event |
||||
if event.Status == gomemcached.NOT_MY_VBUCKET { |
||||
logging.Infof(" Got a not my vbucket error !! ") |
||||
if err := feed.bucket.Refresh(); err != nil { |
||||
logging.Errorf("Unable to refresh bucket %s ", err.Error()) |
||||
feed.closeNodeFeeds() |
||||
return |
||||
} |
||||
// this will only connect to nodes that are not connected or changed
|
||||
// user will have to reconnect the stream
|
||||
if err := feed.connectToNodes(); err != nil { |
||||
logging.Errorf("Unable to connect to nodes %s", err.Error()) |
||||
return |
||||
} |
||||
|
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (feed *UprFeed) closeNodeFeeds() { |
||||
for _, f := range feed.nodeFeeds { |
||||
logging.Infof(" Sending close to forwardUprEvent ") |
||||
close(f.quit) |
||||
f.uprFeed.Close() |
||||
} |
||||
feed.nodeFeeds = nil |
||||
} |
||||
|
||||
// Close a Upr feed.
|
||||
func (feed *UprFeed) Close() error { |
||||
select { |
||||
case <-feed.quit: |
||||
return nil |
||||
default: |
||||
} |
||||
|
||||
feed.closing = true |
||||
feed.closeNodeFeeds() |
||||
close(feed.quit) |
||||
|
||||
feed.wg.Wait() |
||||
if feed.outputClosed == false { |
||||
feed.outputClosed = true |
||||
close(feed.output) |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
@ -0,0 +1,119 @@ |
||||
package couchbase |
||||
|
||||
import ( |
||||
"bytes" |
||||
"fmt" |
||||
) |
||||
|
||||
type User struct { |
||||
Name string |
||||
Id string |
||||
Domain string |
||||
Roles []Role |
||||
} |
||||
|
||||
type Role struct { |
||||
Role string |
||||
BucketName string `json:"bucket_name"` |
||||
} |
||||
|
||||
// Sample:
|
||||
// {"role":"admin","name":"Admin","desc":"Can manage ALL cluster features including security.","ce":true}
|
||||
// {"role":"query_select","bucket_name":"*","name":"Query Select","desc":"Can execute SELECT statement on bucket to retrieve data"}
|
||||
type RoleDescription struct { |
||||
Role string |
||||
Name string |
||||
Desc string |
||||
Ce bool |
||||
BucketName string `json:"bucket_name"` |
||||
} |
||||
|
||||
// Return user-role data, as parsed JSON.
|
||||
// Sample:
|
||||
// [{"id":"ivanivanov","name":"Ivan Ivanov","roles":[{"role":"cluster_admin"},{"bucket_name":"default","role":"bucket_admin"}]},
|
||||
// {"id":"petrpetrov","name":"Petr Petrov","roles":[{"role":"replication_admin"}]}]
|
||||
func (c *Client) GetUserRoles() ([]interface{}, error) { |
||||
ret := make([]interface{}, 0, 1) |
||||
err := c.parseURLResponse("/settings/rbac/users", &ret) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
// Get the configured administrator.
|
||||
// Expected result: {"port":8091,"username":"Administrator"}
|
||||
adminInfo := make(map[string]interface{}, 2) |
||||
err = c.parseURLResponse("/settings/web", &adminInfo) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
// Create a special entry for the configured administrator.
|
||||
adminResult := map[string]interface{}{ |
||||
"name": adminInfo["username"], |
||||
"id": adminInfo["username"], |
||||
"domain": "ns_server", |
||||
"roles": []interface{}{ |
||||
map[string]interface{}{ |
||||
"role": "admin", |
||||
}, |
||||
}, |
||||
} |
||||
|
||||
// Add the configured administrator to the list of results.
|
||||
ret = append(ret, adminResult) |
||||
|
||||
return ret, nil |
||||
} |
||||
|
||||
func (c *Client) GetUserInfoAll() ([]User, error) { |
||||
ret := make([]User, 0, 16) |
||||
err := c.parseURLResponse("/settings/rbac/users", &ret) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return ret, nil |
||||
} |
||||
|
||||
func rolesToParamFormat(roles []Role) string { |
||||
var buffer bytes.Buffer |
||||
for i, role := range roles { |
||||
if i > 0 { |
||||
buffer.WriteString(",") |
||||
} |
||||
buffer.WriteString(role.Role) |
||||
if role.BucketName != "" { |
||||
buffer.WriteString("[") |
||||
buffer.WriteString(role.BucketName) |
||||
buffer.WriteString("]") |
||||
} |
||||
} |
||||
return buffer.String() |
||||
} |
||||
|
||||
func (c *Client) PutUserInfo(u *User) error { |
||||
params := map[string]interface{}{ |
||||
"name": u.Name, |
||||
"roles": rolesToParamFormat(u.Roles), |
||||
} |
||||
var target string |
||||
switch u.Domain { |
||||
case "external": |
||||
target = "/settings/rbac/users/" + u.Id |
||||
case "local": |
||||
target = "/settings/rbac/users/local/" + u.Id |
||||
default: |
||||
return fmt.Errorf("Unknown user type: %s", u.Domain) |
||||
} |
||||
var ret string // PUT returns an empty string. We ignore it.
|
||||
err := c.parsePutURLResponse(target, params, &ret) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) GetRolesAll() ([]RoleDescription, error) { |
||||
ret := make([]RoleDescription, 0, 32) |
||||
err := c.parseURLResponse("/settings/rbac/roles", &ret) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return ret, nil |
||||
} |
||||
@ -0,0 +1,49 @@ |
||||
package couchbase |
||||
|
||||
import ( |
||||
"fmt" |
||||
"net/url" |
||||
"strings" |
||||
) |
||||
|
||||
// CleanupHost returns the hostname with the given suffix removed.
|
||||
func CleanupHost(h, commonSuffix string) string { |
||||
if strings.HasSuffix(h, commonSuffix) { |
||||
return h[:len(h)-len(commonSuffix)] |
||||
} |
||||
return h |
||||
} |
||||
|
||||
// FindCommonSuffix returns the longest common suffix from the given
|
||||
// strings.
|
||||
func FindCommonSuffix(input []string) string { |
||||
rv := "" |
||||
if len(input) < 2 { |
||||
return "" |
||||
} |
||||
from := input |
||||
for i := len(input[0]); i > 0; i-- { |
||||
common := true |
||||
suffix := input[0][i:] |
||||
for _, s := range from { |
||||
if !strings.HasSuffix(s, suffix) { |
||||
common = false |
||||
break |
||||
} |
||||
} |
||||
if common { |
||||
rv = suffix |
||||
} |
||||
} |
||||
return rv |
||||
} |
||||
|
||||
// ParseURL is a wrapper around url.Parse with some sanity-checking
|
||||
func ParseURL(urlStr string) (result *url.URL, err error) { |
||||
result, err = url.Parse(urlStr) |
||||
if result != nil && result.Scheme == "" { |
||||
result = nil |
||||
err = fmt.Errorf("invalid URL <%s>", urlStr) |
||||
} |
||||
return |
||||
} |
||||
@ -0,0 +1,77 @@ |
||||
package couchbase |
||||
|
||||
var crc32tab = []uint32{ |
||||
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, |
||||
0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, |
||||
0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, |
||||
0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, |
||||
0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, |
||||
0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, |
||||
0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, |
||||
0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, |
||||
0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, |
||||
0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, |
||||
0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, |
||||
0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, |
||||
0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, |
||||
0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, |
||||
0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, |
||||
0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, |
||||
0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, |
||||
0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, |
||||
0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, |
||||
0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, |
||||
0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, |
||||
0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, |
||||
0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, |
||||
0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, |
||||
0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, |
||||
0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, |
||||
0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, |
||||
0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, |
||||
0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, |
||||
0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, |
||||
0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, |
||||
0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, |
||||
0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, |
||||
0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, |
||||
0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, |
||||
0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, |
||||
0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, |
||||
0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, |
||||
0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, |
||||
0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, |
||||
0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, |
||||
0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, |
||||
0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, |
||||
0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, |
||||
0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, |
||||
0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, |
||||
0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, |
||||
0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, |
||||
0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, |
||||
0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, |
||||
0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, |
||||
0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, |
||||
0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, |
||||
0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, |
||||
0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, |
||||
0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, |
||||
0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, |
||||
0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, |
||||
0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, |
||||
0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, |
||||
0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, |
||||
0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, |
||||
0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, |
||||
0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d} |
||||
|
||||
// VBHash finds the vbucket for the given key.
|
||||
func (b *Bucket) VBHash(key string) uint32 { |
||||
crc := uint32(0xffffffff) |
||||
for x := 0; x < len(key); x++ { |
||||
crc = (crc >> 8) ^ crc32tab[(uint64(crc)^uint64(key[x]))&0xff] |
||||
} |
||||
vbm := b.VBServerMap() |
||||
return ((^crc) >> 16) & 0x7fff & (uint32(len(vbm.VBucketMap)) - 1) |
||||
} |
||||
@ -0,0 +1,231 @@ |
||||
package couchbase |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"errors" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"math/rand" |
||||
"net/http" |
||||
"net/url" |
||||
"time" |
||||
) |
||||
|
||||
// ViewRow represents a single result from a view.
|
||||
//
|
||||
// Doc is present only if include_docs was set on the request.
|
||||
type ViewRow struct { |
||||
ID string |
||||
Key interface{} |
||||
Value interface{} |
||||
Doc *interface{} |
||||
} |
||||
|
||||
// A ViewError is a node-specific error indicating a partial failure
|
||||
// within a view result.
|
||||
type ViewError struct { |
||||
From string |
||||
Reason string |
||||
} |
||||
|
||||
func (ve ViewError) Error() string { |
||||
return "Node: " + ve.From + ", reason: " + ve.Reason |
||||
} |
||||
|
||||
// ViewResult holds the entire result set from a view request,
|
||||
// including the rows and the errors.
|
||||
type ViewResult struct { |
||||
TotalRows int `json:"total_rows"` |
||||
Rows []ViewRow |
||||
Errors []ViewError |
||||
} |
||||
|
||||
func (b *Bucket) randomBaseURL() (*url.URL, error) { |
||||
nodes := b.HealthyNodes() |
||||
if len(nodes) == 0 { |
||||
return nil, errors.New("no available couch rest URLs") |
||||
} |
||||
nodeNo := rand.Intn(len(nodes)) |
||||
node := nodes[nodeNo] |
||||
|
||||
b.RLock() |
||||
name := b.Name |
||||
pool := b.pool |
||||
b.RUnlock() |
||||
|
||||
u, err := ParseURL(node.CouchAPIBase) |
||||
if err != nil { |
||||
return nil, fmt.Errorf("config error: Bucket %q node #%d CouchAPIBase=%q: %v", |
||||
name, nodeNo, node.CouchAPIBase, err) |
||||
} else if pool != nil { |
||||
u.User = pool.client.BaseURL.User |
||||
} |
||||
return u, err |
||||
} |
||||
|
||||
const START_NODE_ID = -1 |
||||
|
||||
func (b *Bucket) randomNextURL(lastNode int) (*url.URL, int, error) { |
||||
nodes := b.HealthyNodes() |
||||
if len(nodes) == 0 { |
||||
return nil, -1, errors.New("no available couch rest URLs") |
||||
} |
||||
|
||||
var nodeNo int |
||||
if lastNode == START_NODE_ID || lastNode >= len(nodes) { |
||||
// randomly select a node if the value of lastNode is invalid
|
||||
nodeNo = rand.Intn(len(nodes)) |
||||
} else { |
||||
// wrap around the node list
|
||||
nodeNo = (lastNode + 1) % len(nodes) |
||||
} |
||||
|
||||
b.RLock() |
||||
name := b.Name |
||||
pool := b.pool |
||||
b.RUnlock() |
||||
|
||||
node := nodes[nodeNo] |
||||
u, err := ParseURL(node.CouchAPIBase) |
||||
if err != nil { |
||||
return nil, -1, fmt.Errorf("config error: Bucket %q node #%d CouchAPIBase=%q: %v", |
||||
name, nodeNo, node.CouchAPIBase, err) |
||||
} else if pool != nil { |
||||
u.User = pool.client.BaseURL.User |
||||
} |
||||
return u, nodeNo, err |
||||
} |
||||
|
||||
// DocID is the document ID type for the startkey_docid parameter in
|
||||
// views.
|
||||
type DocID string |
||||
|
||||
func qParam(k, v string) string { |
||||
format := `"%s"` |
||||
switch k { |
||||
case "startkey_docid", "endkey_docid", "stale": |
||||
format = "%s" |
||||
} |
||||
return fmt.Sprintf(format, v) |
||||
} |
||||
|
||||
// ViewURL constructs a URL for a view with the given ddoc, view name,
|
||||
// and parameters.
|
||||
func (b *Bucket) ViewURL(ddoc, name string, |
||||
params map[string]interface{}) (string, error) { |
||||
u, err := b.randomBaseURL() |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
|
||||
values := url.Values{} |
||||
for k, v := range params { |
||||
switch t := v.(type) { |
||||
case DocID: |
||||
values[k] = []string{string(t)} |
||||
case string: |
||||
values[k] = []string{qParam(k, t)} |
||||
case int: |
||||
values[k] = []string{fmt.Sprintf(`%d`, t)} |
||||
case bool: |
||||
values[k] = []string{fmt.Sprintf(`%v`, t)} |
||||
default: |
||||
b, err := json.Marshal(v) |
||||
if err != nil { |
||||
return "", fmt.Errorf("unsupported value-type %T in Query, "+ |
||||
"json encoder said %v", t, err) |
||||
} |
||||
values[k] = []string{fmt.Sprintf(`%v`, string(b))} |
||||
} |
||||
} |
||||
|
||||
if ddoc == "" && name == "_all_docs" { |
||||
u.Path = fmt.Sprintf("/%s/_all_docs", b.GetName()) |
||||
} else { |
||||
u.Path = fmt.Sprintf("/%s/_design/%s/_view/%s", b.GetName(), ddoc, name) |
||||
} |
||||
u.RawQuery = values.Encode() |
||||
|
||||
return u.String(), nil |
||||
} |
||||
|
||||
// ViewCallback is called for each view invocation.
|
||||
var ViewCallback func(ddoc, name string, start time.Time, err error) |
||||
|
||||
// ViewCustom performs a view request that can map row values to a
|
||||
// custom type.
|
||||
//
|
||||
// See the source to View for an example usage.
|
||||
func (b *Bucket) ViewCustom(ddoc, name string, params map[string]interface{}, |
||||
vres interface{}) (err error) { |
||||
if SlowServerCallWarningThreshold > 0 { |
||||
defer slowLog(time.Now(), "call to ViewCustom(%q, %q)", ddoc, name) |
||||
} |
||||
|
||||
if ViewCallback != nil { |
||||
defer func(t time.Time) { ViewCallback(ddoc, name, t, err) }(time.Now()) |
||||
} |
||||
|
||||
u, err := b.ViewURL(ddoc, name, params) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
req, err := http.NewRequest("GET", u, nil) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
ah := b.authHandler(false /* bucket not yet locked */) |
||||
maybeAddAuth(req, ah) |
||||
|
||||
res, err := doHTTPRequest(req) |
||||
if err != nil { |
||||
return fmt.Errorf("error starting view req at %v: %v", u, err) |
||||
} |
||||
defer res.Body.Close() |
||||
|
||||
if res.StatusCode != 200 { |
||||
bod := make([]byte, 512) |
||||
l, _ := res.Body.Read(bod) |
||||
return fmt.Errorf("error executing view req at %v: %v - %s", |
||||
u, res.Status, bod[:l]) |
||||
} |
||||
|
||||
body, err := ioutil.ReadAll(res.Body) |
||||
if err := json.Unmarshal(body, vres); err != nil { |
||||
return nil |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// View executes a view.
|
||||
//
|
||||
// The ddoc parameter is just the bare name of your design doc without
|
||||
// the "_design/" prefix.
|
||||
//
|
||||
// Parameters are string keys with values that correspond to couchbase
|
||||
// view parameters. Primitive should work fairly naturally (booleans,
|
||||
// ints, strings, etc...) and other values will attempt to be JSON
|
||||
// marshaled (useful for array indexing on on view keys, for example).
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
// res, err := couchbase.View("myddoc", "myview", map[string]interface{}{
|
||||
// "group_level": 2,
|
||||
// "startkey_docid": []interface{}{"thing"},
|
||||
// "endkey_docid": []interface{}{"thing", map[string]string{}},
|
||||
// "stale": false,
|
||||
// })
|
||||
func (b *Bucket) View(ddoc, name string, params map[string]interface{}) (ViewResult, error) { |
||||
vres := ViewResult{} |
||||
|
||||
if err := b.ViewCustom(ddoc, name, params, &vres); err != nil { |
||||
//error in accessing views. Retry once after a bucket refresh
|
||||
b.Refresh() |
||||
return vres, b.ViewCustom(ddoc, name, params, &vres) |
||||
} else { |
||||
return vres, nil |
||||
} |
||||
} |
||||
@ -0,0 +1,228 @@ |
||||
// Copyright 2013 Beego Authors
|
||||
// Copyright 2014 The Macaron Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"): you may
|
||||
// not use this file except in compliance with the License. You may obtain
|
||||
// a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
// License for the specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package session |
||||
|
||||
import ( |
||||
"strings" |
||||
"sync" |
||||
|
||||
"github.com/couchbaselabs/go-couchbase" |
||||
|
||||
"github.com/go-macaron/session" |
||||
) |
||||
|
||||
// CouchbaseSessionStore represents a couchbase session store implementation.
|
||||
type CouchbaseSessionStore struct { |
||||
b *couchbase.Bucket |
||||
sid string |
||||
lock sync.RWMutex |
||||
data map[interface{}]interface{} |
||||
maxlifetime int64 |
||||
} |
||||
|
||||
// Set sets value to given key in session.
|
||||
func (s *CouchbaseSessionStore) Set(key, val interface{}) error { |
||||
s.lock.Lock() |
||||
defer s.lock.Unlock() |
||||
|
||||
s.data[key] = val |
||||
return nil |
||||
} |
||||
|
||||
// Get gets value by given key in session.
|
||||
func (s *CouchbaseSessionStore) Get(key interface{}) interface{} { |
||||
s.lock.RLock() |
||||
defer s.lock.RUnlock() |
||||
|
||||
return s.data[key] |
||||
} |
||||
|
||||
// Delete delete a key from session.
|
||||
func (s *CouchbaseSessionStore) Delete(key interface{}) error { |
||||
s.lock.Lock() |
||||
defer s.lock.Unlock() |
||||
|
||||
delete(s.data, key) |
||||
return nil |
||||
} |
||||
|
||||
// ID returns current session ID.
|
||||
func (s *CouchbaseSessionStore) ID() string { |
||||
return s.sid |
||||
} |
||||
|
||||
// Release releases resource and save data to provider.
|
||||
func (s *CouchbaseSessionStore) Release() error { |
||||
defer s.b.Close() |
||||
|
||||
// Skip encoding if the data is empty
|
||||
if len(s.data) == 0 { |
||||
return nil |
||||
} |
||||
|
||||
data, err := session.EncodeGob(s.data) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
return s.b.Set(s.sid, int(s.maxlifetime), data) |
||||
} |
||||
|
||||
// Flush deletes all session data.
|
||||
func (s *CouchbaseSessionStore) Flush() error { |
||||
s.lock.Lock() |
||||
defer s.lock.Unlock() |
||||
|
||||
s.data = make(map[interface{}]interface{}) |
||||
return nil |
||||
} |
||||
|
||||
// CouchbaseProvider represents a couchbase session provider implementation.
|
||||
type CouchbaseProvider struct { |
||||
maxlifetime int64 |
||||
connStr string |
||||
pool string |
||||
bucket string |
||||
b *couchbase.Bucket |
||||
} |
||||
|
||||
func (cp *CouchbaseProvider) getBucket() *couchbase.Bucket { |
||||
c, err := couchbase.Connect(cp.connStr) |
||||
if err != nil { |
||||
return nil |
||||
} |
||||
|
||||
pool, err := c.GetPool(cp.pool) |
||||
if err != nil { |
||||
return nil |
||||
} |
||||
|
||||
bucket, err := pool.GetBucket(cp.bucket) |
||||
if err != nil { |
||||
return nil |
||||
} |
||||
|
||||
return bucket |
||||
} |
||||
|
||||
// Init initializes memory session provider.
|
||||
// connStr is couchbase server REST/JSON URL
|
||||
// e.g. http://host:port/, Pool, Bucket
|
||||
func (p *CouchbaseProvider) Init(maxlifetime int64, connStr string) error { |
||||
p.maxlifetime = maxlifetime |
||||
configs := strings.Split(connStr, ",") |
||||
if len(configs) > 0 { |
||||
p.connStr = configs[0] |
||||
} |
||||
if len(configs) > 1 { |
||||
p.pool = configs[1] |
||||
} |
||||
if len(configs) > 2 { |
||||
p.bucket = configs[2] |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// Read returns raw session store by session ID.
|
||||
func (p *CouchbaseProvider) Read(sid string) (session.RawStore, error) { |
||||
p.b = p.getBucket() |
||||
|
||||
var doc []byte |
||||
|
||||
err := p.b.Get(sid, &doc) |
||||
var kv map[interface{}]interface{} |
||||
if doc == nil { |
||||
kv = make(map[interface{}]interface{}) |
||||
} else { |
||||
kv, err = session.DecodeGob(doc) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
cs := &CouchbaseSessionStore{b: p.b, sid: sid, data: kv, maxlifetime: p.maxlifetime} |
||||
return cs, nil |
||||
} |
||||
|
||||
// Exist returns true if session with given ID exists.
|
||||
func (p *CouchbaseProvider) Exist(sid string) bool { |
||||
p.b = p.getBucket() |
||||
defer p.b.Close() |
||||
|
||||
var doc []byte |
||||
|
||||
if err := p.b.Get(sid, &doc); err != nil || doc == nil { |
||||
return false |
||||
} else { |
||||
return true |
||||
} |
||||
} |
||||
|
||||
// Destory deletes a session by session ID.
|
||||
func (p *CouchbaseProvider) Destory(sid string) error { |
||||
p.b = p.getBucket() |
||||
defer p.b.Close() |
||||
|
||||
p.b.Delete(sid) |
||||
return nil |
||||
} |
||||
|
||||
// Regenerate regenerates a session store from old session ID to new one.
|
||||
func (p *CouchbaseProvider) Regenerate(oldsid, sid string) (session.RawStore, error) { |
||||
p.b = p.getBucket() |
||||
|
||||
var doc []byte |
||||
if err := p.b.Get(oldsid, &doc); err != nil || doc == nil { |
||||
p.b.Set(sid, int(p.maxlifetime), "") |
||||
} else { |
||||
err := p.b.Delete(oldsid) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
_, _ = p.b.Add(sid, int(p.maxlifetime), doc) |
||||
} |
||||
|
||||
err := p.b.Get(sid, &doc) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
var kv map[interface{}]interface{} |
||||
if doc == nil { |
||||
kv = make(map[interface{}]interface{}) |
||||
} else { |
||||
kv, err = session.DecodeGob(doc) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
cs := &CouchbaseSessionStore{b: p.b, sid: sid, data: kv, maxlifetime: p.maxlifetime} |
||||
return cs, nil |
||||
} |
||||
|
||||
// Count counts and returns number of sessions.
|
||||
func (p *CouchbaseProvider) Count() int { |
||||
// FIXME
|
||||
return 0 |
||||
} |
||||
|
||||
// GC calls GC to clean expired sessions.
|
||||
func (p *CouchbaseProvider) GC() {} |
||||
|
||||
func init() { |
||||
session.Register("couchbase", &CouchbaseProvider{}) |
||||
} |
||||
@ -0,0 +1,61 @@ |
||||
// Copyright 2018 The Macaron Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"): you may
|
||||
// not use this file except in compliance with the License. You may obtain
|
||||
// a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
// License for the specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package session |
||||
|
||||
import ( |
||||
"net/url" |
||||
|
||||
"gopkg.in/macaron.v1" |
||||
) |
||||
|
||||
type Flash struct { |
||||
ctx *macaron.Context |
||||
url.Values |
||||
ErrorMsg, WarningMsg, InfoMsg, SuccessMsg string |
||||
} |
||||
|
||||
func (f *Flash) set(name, msg string, current ...bool) { |
||||
isShow := false |
||||
if (len(current) == 0 && macaron.FlashNow) || |
||||
(len(current) > 0 && current[0]) { |
||||
isShow = true |
||||
} |
||||
|
||||
if isShow { |
||||
f.ctx.Data["Flash"] = f |
||||
} else { |
||||
f.Set(name, msg) |
||||
} |
||||
} |
||||
|
||||
func (f *Flash) Error(msg string, current ...bool) { |
||||
f.ErrorMsg = msg |
||||
f.set("error", msg, current...) |
||||
} |
||||
|
||||
func (f *Flash) Warning(msg string, current ...bool) { |
||||
f.WarningMsg = msg |
||||
f.set("warning", msg, current...) |
||||
} |
||||
|
||||
func (f *Flash) Info(msg string, current ...bool) { |
||||
f.InfoMsg = msg |
||||
f.set("info", msg, current...) |
||||
} |
||||
|
||||
func (f *Flash) Success(msg string, current ...bool) { |
||||
f.SuccessMsg = msg |
||||
f.set("success", msg, current...) |
||||
} |
||||
@ -0,0 +1,204 @@ |
||||
// Copyright 2013 Beego Authors
|
||||
// Copyright 2014 The Macaron Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"): you may
|
||||
// not use this file except in compliance with the License. You may obtain
|
||||
// a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
// License for the specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package session |
||||
|
||||
import ( |
||||
"fmt" |
||||
"strings" |
||||
"sync" |
||||
|
||||
"github.com/bradfitz/gomemcache/memcache" |
||||
|
||||
"github.com/go-macaron/session" |
||||
) |
||||
|
||||
// MemcacheStore represents a memcache session store implementation.
|
||||
type MemcacheStore struct { |
||||
c *memcache.Client |
||||
sid string |
||||
expire int32 |
||||
lock sync.RWMutex |
||||
data map[interface{}]interface{} |
||||
} |
||||
|
||||
// NewMemcacheStore creates and returns a memcache session store.
|
||||
func NewMemcacheStore(c *memcache.Client, sid string, expire int32, kv map[interface{}]interface{}) *MemcacheStore { |
||||
return &MemcacheStore{ |
||||
c: c, |
||||
sid: sid, |
||||
expire: expire, |
||||
data: kv, |
||||
} |
||||
} |
||||
|
||||
func NewItem(sid string, data []byte, expire int32) *memcache.Item { |
||||
return &memcache.Item{ |
||||
Key: sid, |
||||
Value: data, |
||||
Expiration: expire, |
||||
} |
||||
} |
||||
|
||||
// Set sets value to given key in session.
|
||||
func (s *MemcacheStore) Set(key, val interface{}) error { |
||||
s.lock.Lock() |
||||
defer s.lock.Unlock() |
||||
|
||||
s.data[key] = val |
||||
return nil |
||||
} |
||||
|
||||
// Get gets value by given key in session.
|
||||
func (s *MemcacheStore) Get(key interface{}) interface{} { |
||||
s.lock.RLock() |
||||
defer s.lock.RUnlock() |
||||
|
||||
return s.data[key] |
||||
} |
||||
|
||||
// Delete delete a key from session.
|
||||
func (s *MemcacheStore) Delete(key interface{}) error { |
||||
s.lock.Lock() |
||||
defer s.lock.Unlock() |
||||
|
||||
delete(s.data, key) |
||||
return nil |
||||
} |
||||
|
||||
// ID returns current session ID.
|
||||
func (s *MemcacheStore) ID() string { |
||||
return s.sid |
||||
} |
||||
|
||||
// Release releases resource and save data to provider.
|
||||
func (s *MemcacheStore) Release() error { |
||||
// Skip encoding if the data is empty
|
||||
if len(s.data) == 0 { |
||||
return nil |
||||
} |
||||
|
||||
data, err := session.EncodeGob(s.data) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
return s.c.Set(NewItem(s.sid, data, s.expire)) |
||||
} |
||||
|
||||
// Flush deletes all session data.
|
||||
func (s *MemcacheStore) Flush() error { |
||||
s.lock.Lock() |
||||
defer s.lock.Unlock() |
||||
|
||||
s.data = make(map[interface{}]interface{}) |
||||
return nil |
||||
} |
||||
|
||||
// MemcacheProvider represents a memcache session provider implementation.
|
||||
type MemcacheProvider struct { |
||||
c *memcache.Client |
||||
expire int32 |
||||
} |
||||
|
||||
// Init initializes memcache session provider.
|
||||
// connStrs: 127.0.0.1:9090;127.0.0.1:9091
|
||||
func (p *MemcacheProvider) Init(expire int64, connStrs string) error { |
||||
p.expire = int32(expire) |
||||
p.c = memcache.New(strings.Split(connStrs, ";")...) |
||||
return nil |
||||
} |
||||
|
||||
// Read returns raw session store by session ID.
|
||||
func (p *MemcacheProvider) Read(sid string) (session.RawStore, error) { |
||||
if !p.Exist(sid) { |
||||
if err := p.c.Set(NewItem(sid, []byte(""), p.expire)); err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
var kv map[interface{}]interface{} |
||||
item, err := p.c.Get(sid) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if len(item.Value) == 0 { |
||||
kv = make(map[interface{}]interface{}) |
||||
} else { |
||||
kv, err = session.DecodeGob(item.Value) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
return NewMemcacheStore(p.c, sid, p.expire, kv), nil |
||||
} |
||||
|
||||
// Exist returns true if session with given ID exists.
|
||||
func (p *MemcacheProvider) Exist(sid string) bool { |
||||
_, err := p.c.Get(sid) |
||||
return err == nil |
||||
} |
||||
|
||||
// Destory deletes a session by session ID.
|
||||
func (p *MemcacheProvider) Destory(sid string) error { |
||||
return p.c.Delete(sid) |
||||
} |
||||
|
||||
// Regenerate regenerates a session store from old session ID to new one.
|
||||
func (p *MemcacheProvider) Regenerate(oldsid, sid string) (_ session.RawStore, err error) { |
||||
if p.Exist(sid) { |
||||
return nil, fmt.Errorf("new sid '%s' already exists", sid) |
||||
} |
||||
|
||||
item := NewItem(sid, []byte(""), p.expire) |
||||
if p.Exist(oldsid) { |
||||
item, err = p.c.Get(oldsid) |
||||
if err != nil { |
||||
return nil, err |
||||
} else if err = p.c.Delete(oldsid); err != nil { |
||||
return nil, err |
||||
} |
||||
item.Key = sid |
||||
} |
||||
if err = p.c.Set(item); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
var kv map[interface{}]interface{} |
||||
if len(item.Value) == 0 { |
||||
kv = make(map[interface{}]interface{}) |
||||
} else { |
||||
kv, err = session.DecodeGob(item.Value) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
return NewMemcacheStore(p.c, sid, p.expire, kv), nil |
||||
} |
||||
|
||||
// Count counts and returns number of sessions.
|
||||
func (p *MemcacheProvider) Count() int { |
||||
// FIXME: how come this library does not have Stats method?
|
||||
return -1 |
||||
} |
||||
|
||||
// GC calls GC to clean expired sessions.
|
||||
func (p *MemcacheProvider) GC() {} |
||||
|
||||
func init() { |
||||
session.Register("memcache", &MemcacheProvider{}) |
||||
} |
||||
@ -0,0 +1,200 @@ |
||||
// Copyright 2013 Beego Authors
|
||||
// Copyright 2014 The Macaron Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"): you may
|
||||
// not use this file except in compliance with the License. You may obtain
|
||||
// a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
// License for the specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package session |
||||
|
||||
import ( |
||||
"database/sql" |
||||
"fmt" |
||||
"log" |
||||
"sync" |
||||
"time" |
||||
|
||||
_ "github.com/go-sql-driver/mysql" |
||||
|
||||
"github.com/go-macaron/session" |
||||
) |
||||
|
||||
// MysqlStore represents a mysql session store implementation.
|
||||
type MysqlStore struct { |
||||
c *sql.DB |
||||
sid string |
||||
lock sync.RWMutex |
||||
data map[interface{}]interface{} |
||||
} |
||||
|
||||
// NewMysqlStore creates and returns a mysql session store.
|
||||
func NewMysqlStore(c *sql.DB, sid string, kv map[interface{}]interface{}) *MysqlStore { |
||||
return &MysqlStore{ |
||||
c: c, |
||||
sid: sid, |
||||
data: kv, |
||||
} |
||||
} |
||||
|
||||
// Set sets value to given key in session.
|
||||
func (s *MysqlStore) Set(key, val interface{}) error { |
||||
s.lock.Lock() |
||||
defer s.lock.Unlock() |
||||
|
||||
s.data[key] = val |
||||
return nil |
||||
} |
||||
|
||||
// Get gets value by given key in session.
|
||||
func (s *MysqlStore) Get(key interface{}) interface{} { |
||||
s.lock.RLock() |
||||
defer s.lock.RUnlock() |
||||
|
||||
return s.data[key] |
||||
} |
||||
|
||||
// Delete delete a key from session.
|
||||
func (s *MysqlStore) Delete(key interface{}) error { |
||||
s.lock.Lock() |
||||
defer s.lock.Unlock() |
||||
|
||||
delete(s.data, key) |
||||
return nil |
||||
} |
||||
|
||||
// ID returns current session ID.
|
||||
func (s *MysqlStore) ID() string { |
||||
return s.sid |
||||
} |
||||
|
||||
// Release releases resource and save data to provider.
|
||||
func (s *MysqlStore) Release() error { |
||||
// Skip encoding if the data is empty
|
||||
if len(s.data) == 0 { |
||||
return nil |
||||
} |
||||
|
||||
data, err := session.EncodeGob(s.data) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
_, err = s.c.Exec("UPDATE session SET data=?, expiry=? WHERE `key`=?", |
||||
data, time.Now().Unix(), s.sid) |
||||
return err |
||||
} |
||||
|
||||
// Flush deletes all session data.
|
||||
func (s *MysqlStore) Flush() error { |
||||
s.lock.Lock() |
||||
defer s.lock.Unlock() |
||||
|
||||
s.data = make(map[interface{}]interface{}) |
||||
return nil |
||||
} |
||||
|
||||
// MysqlProvider represents a mysql session provider implementation.
|
||||
type MysqlProvider struct { |
||||
c *sql.DB |
||||
expire int64 |
||||
} |
||||
|
||||
// Init initializes mysql session provider.
|
||||
// connStr: username:password@protocol(address)/dbname?param=value
|
||||
func (p *MysqlProvider) Init(expire int64, connStr string) (err error) { |
||||
p.expire = expire |
||||
|
||||
p.c, err = sql.Open("mysql", connStr) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return p.c.Ping() |
||||
} |
||||
|
||||
// Read returns raw session store by session ID.
|
||||
func (p *MysqlProvider) Read(sid string) (session.RawStore, error) { |
||||
var data []byte |
||||
err := p.c.QueryRow("SELECT data FROM session WHERE `key`=?", sid).Scan(&data) |
||||
if err == sql.ErrNoRows { |
||||
_, err = p.c.Exec("INSERT INTO session(`key`,data,expiry) VALUES(?,?,?)", |
||||
sid, "", time.Now().Unix()) |
||||
} |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
var kv map[interface{}]interface{} |
||||
if len(data) == 0 { |
||||
kv = make(map[interface{}]interface{}) |
||||
} else { |
||||
kv, err = session.DecodeGob(data) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
return NewMysqlStore(p.c, sid, kv), nil |
||||
} |
||||
|
||||
// Exist returns true if session with given ID exists.
|
||||
func (p *MysqlProvider) Exist(sid string) bool { |
||||
var data []byte |
||||
err := p.c.QueryRow("SELECT data FROM session WHERE `key`=?", sid).Scan(&data) |
||||
if err != nil && err != sql.ErrNoRows { |
||||
panic("session/mysql: error checking existence: " + err.Error()) |
||||
} |
||||
return err != sql.ErrNoRows |
||||
} |
||||
|
||||
// Destory deletes a session by session ID.
|
||||
func (p *MysqlProvider) Destory(sid string) error { |
||||
_, err := p.c.Exec("DELETE FROM session WHERE `key`=?", sid) |
||||
return err |
||||
} |
||||
|
||||
// Regenerate regenerates a session store from old session ID to new one.
|
||||
func (p *MysqlProvider) Regenerate(oldsid, sid string) (_ session.RawStore, err error) { |
||||
if p.Exist(sid) { |
||||
return nil, fmt.Errorf("new sid '%s' already exists", sid) |
||||
} |
||||
|
||||
if !p.Exist(oldsid) { |
||||
if _, err = p.c.Exec("INSERT INTO session(`key`,data,expiry) VALUES(?,?,?)", |
||||
oldsid, "", time.Now().Unix()); err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
if _, err = p.c.Exec("UPDATE session SET `key`=? WHERE `key`=?", sid, oldsid); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return p.Read(sid) |
||||
} |
||||
|
||||
// Count counts and returns number of sessions.
|
||||
func (p *MysqlProvider) Count() (total int) { |
||||
if err := p.c.QueryRow("SELECT COUNT(*) AS NUM FROM session").Scan(&total); err != nil { |
||||
panic("session/mysql: error counting records: " + err.Error()) |
||||
} |
||||
return total |
||||
} |
||||
|
||||
// GC calls GC to clean expired sessions.
|
||||
func (p *MysqlProvider) GC() { |
||||
if _, err := p.c.Exec("DELETE FROM session WHERE expiry + ? <= UNIX_TIMESTAMP(NOW())", p.expire); err != nil { |
||||
log.Printf("session/mysql: error garbage collecting: %v", err) |
||||
} |
||||
} |
||||
|
||||
func init() { |
||||
session.Register("mysql", &MysqlProvider{}) |
||||
} |
||||
@ -0,0 +1,208 @@ |
||||
// Copyright 2015 The Macaron Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"): you may
|
||||
// not use this file except in compliance with the License. You may obtain
|
||||
// a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
// License for the specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package session |
||||
|
||||
import ( |
||||
"fmt" |
||||
"sync" |
||||
|
||||
"github.com/lunny/nodb" |
||||
"github.com/lunny/nodb/config" |
||||
|
||||
"github.com/go-macaron/session" |
||||
) |
||||
|
||||
// NodbStore represents a nodb session store implementation.
|
||||
type NodbStore struct { |
||||
c *nodb.DB |
||||
sid string |
||||
expire int64 |
||||
lock sync.RWMutex |
||||
data map[interface{}]interface{} |
||||
} |
||||
|
||||
// NewNodbStore creates and returns a ledis session store.
|
||||
func NewNodbStore(c *nodb.DB, sid string, expire int64, kv map[interface{}]interface{}) *NodbStore { |
||||
return &NodbStore{ |
||||
c: c, |
||||
expire: expire, |
||||
sid: sid, |
||||
data: kv, |
||||
} |
||||
} |
||||
|
||||
// Set sets value to given key in session.
|
||||
func (s *NodbStore) Set(key, val interface{}) error { |
||||
s.lock.Lock() |
||||
defer s.lock.Unlock() |
||||
|
||||
s.data[key] = val |
||||
return nil |
||||
} |
||||
|
||||
// Get gets value by given key in session.
|
||||
func (s *NodbStore) Get(key interface{}) interface{} { |
||||
s.lock.RLock() |
||||
defer s.lock.RUnlock() |
||||
|
||||
return s.data[key] |
||||
} |
||||
|
||||
// Delete delete a key from session.
|
||||
func (s *NodbStore) Delete(key interface{}) error { |
||||
s.lock.Lock() |
||||
defer s.lock.Unlock() |
||||
|
||||
delete(s.data, key) |
||||
return nil |
||||
} |
||||
|
||||
// ID returns current session ID.
|
||||
func (s *NodbStore) ID() string { |
||||
return s.sid |
||||
} |
||||
|
||||
// Release releases resource and save data to provider.
|
||||
func (s *NodbStore) Release() error { |
||||
// Skip encoding if the data is empty
|
||||
if len(s.data) == 0 { |
||||
return nil |
||||
} |
||||
|
||||
data, err := session.EncodeGob(s.data) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if err = s.c.Set([]byte(s.sid), data); err != nil { |
||||
return err |
||||
} |
||||
_, err = s.c.Expire([]byte(s.sid), s.expire) |
||||
return err |
||||
} |
||||
|
||||
// Flush deletes all session data.
|
||||
func (s *NodbStore) Flush() error { |
||||
s.lock.Lock() |
||||
defer s.lock.Unlock() |
||||
|
||||
s.data = make(map[interface{}]interface{}) |
||||
return nil |
||||
} |
||||
|
||||
// NodbProvider represents a ledis session provider implementation.
|
||||
type NodbProvider struct { |
||||
c *nodb.DB |
||||
expire int64 |
||||
} |
||||
|
||||
// Init initializes nodb session provider.
|
||||
func (p *NodbProvider) Init(expire int64, configs string) error { |
||||
p.expire = expire |
||||
|
||||
cfg := new(config.Config) |
||||
cfg.DataDir = configs |
||||
dbs, err := nodb.Open(cfg) |
||||
if err != nil { |
||||
return fmt.Errorf("session/nodb: error opening db: %v", err) |
||||
} |
||||
|
||||
p.c, err = dbs.Select(0) |
||||
return err |
||||
} |
||||
|
||||
// Read returns raw session store by session ID.
|
||||
func (p *NodbProvider) Read(sid string) (session.RawStore, error) { |
||||
if !p.Exist(sid) { |
||||
if err := p.c.Set([]byte(sid), []byte("")); err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
var kv map[interface{}]interface{} |
||||
kvs, err := p.c.Get([]byte(sid)) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if len(kvs) == 0 { |
||||
kv = make(map[interface{}]interface{}) |
||||
} else { |
||||
kv, err = session.DecodeGob(kvs) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
return NewNodbStore(p.c, sid, p.expire, kv), nil |
||||
} |
||||
|
||||
// Exist returns true if session with given ID exists.
|
||||
func (p *NodbProvider) Exist(sid string) bool { |
||||
count, err := p.c.Exists([]byte(sid)) |
||||
return err == nil && count > 0 |
||||
} |
||||
|
||||
// Destory deletes a session by session ID.
|
||||
func (p *NodbProvider) Destory(sid string) error { |
||||
_, err := p.c.Del([]byte(sid)) |
||||
return err |
||||
} |
||||
|
||||
// Regenerate regenerates a session store from old session ID to new one.
|
||||
func (p *NodbProvider) Regenerate(oldsid, sid string) (_ session.RawStore, err error) { |
||||
if p.Exist(sid) { |
||||
return nil, fmt.Errorf("new sid '%s' already exists", sid) |
||||
} |
||||
|
||||
kvs := make([]byte, 0) |
||||
if p.Exist(oldsid) { |
||||
if kvs, err = p.c.Get([]byte(oldsid)); err != nil { |
||||
return nil, err |
||||
} else if _, err = p.c.Del([]byte(oldsid)); err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
if err = p.c.Set([]byte(sid), kvs); err != nil { |
||||
return nil, err |
||||
} else if _, err = p.c.Expire([]byte(sid), p.expire); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
var kv map[interface{}]interface{} |
||||
if len(kvs) == 0 { |
||||
kv = make(map[interface{}]interface{}) |
||||
} else { |
||||
kv, err = session.DecodeGob([]byte(kvs)) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
return NewNodbStore(p.c, sid, p.expire, kv), nil |
||||
} |
||||
|
||||
// Count counts and returns number of sessions.
|
||||
func (p *NodbProvider) Count() int { |
||||
// FIXME: how come this library does not have DbSize() method?
|
||||
return -1 |
||||
} |
||||
|
||||
// GC calls GC to clean expired sessions.
|
||||
func (p *NodbProvider) GC() {} |
||||
|
||||
func init() { |
||||
session.Register("nodb", &NodbProvider{}) |
||||
} |
||||
@ -0,0 +1,201 @@ |
||||
// Copyright 2013 Beego Authors
|
||||
// Copyright 2014 The Macaron Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"): you may
|
||||
// not use this file except in compliance with the License. You may obtain
|
||||
// a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
// License for the specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package session |
||||
|
||||
import ( |
||||
"database/sql" |
||||
"fmt" |
||||
"log" |
||||
"sync" |
||||
"time" |
||||
|
||||
_ "github.com/lib/pq" |
||||
|
||||
"github.com/go-macaron/session" |
||||
) |
||||
|
||||
// PostgresStore represents a postgres session store implementation.
|
||||
type PostgresStore struct { |
||||
c *sql.DB |
||||
sid string |
||||
lock sync.RWMutex |
||||
data map[interface{}]interface{} |
||||
} |
||||
|
||||
// NewPostgresStore creates and returns a postgres session store.
|
||||
func NewPostgresStore(c *sql.DB, sid string, kv map[interface{}]interface{}) *PostgresStore { |
||||
return &PostgresStore{ |
||||
c: c, |
||||
sid: sid, |
||||
data: kv, |
||||
} |
||||
} |
||||
|
||||
// Set sets value to given key in session.
|
||||
func (s *PostgresStore) Set(key, value interface{}) error { |
||||
s.lock.Lock() |
||||
defer s.lock.Unlock() |
||||
|
||||
s.data[key] = value |
||||
return nil |
||||
} |
||||
|
||||
// Get gets value by given key in session.
|
||||
func (s *PostgresStore) Get(key interface{}) interface{} { |
||||
s.lock.RLock() |
||||
defer s.lock.RUnlock() |
||||
|
||||
return s.data[key] |
||||
} |
||||
|
||||
// Delete delete a key from session.
|
||||
func (s *PostgresStore) Delete(key interface{}) error { |
||||
s.lock.Lock() |
||||
defer s.lock.Unlock() |
||||
|
||||
delete(s.data, key) |
||||
return nil |
||||
} |
||||
|
||||
// ID returns current session ID.
|
||||
func (s *PostgresStore) ID() string { |
||||
return s.sid |
||||
} |
||||
|
||||
// save postgres session values to database.
|
||||
// must call this method to save values to database.
|
||||
func (s *PostgresStore) Release() error { |
||||
// Skip encoding if the data is empty
|
||||
if len(s.data) == 0 { |
||||
return nil |
||||
} |
||||
|
||||
data, err := session.EncodeGob(s.data) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
_, err = s.c.Exec("UPDATE session SET data=$1, expiry=$2 WHERE key=$3", |
||||
data, time.Now().Unix(), s.sid) |
||||
return err |
||||
} |
||||
|
||||
// Flush deletes all session data.
|
||||
func (s *PostgresStore) Flush() error { |
||||
s.lock.Lock() |
||||
defer s.lock.Unlock() |
||||
|
||||
s.data = make(map[interface{}]interface{}) |
||||
return nil |
||||
} |
||||
|
||||
// PostgresProvider represents a postgres session provider implementation.
|
||||
type PostgresProvider struct { |
||||
c *sql.DB |
||||
maxlifetime int64 |
||||
} |
||||
|
||||
// Init initializes postgres session provider.
|
||||
// connStr: user=a password=b host=localhost port=5432 dbname=c sslmode=disable
|
||||
func (p *PostgresProvider) Init(maxlifetime int64, connStr string) (err error) { |
||||
p.maxlifetime = maxlifetime |
||||
|
||||
p.c, err = sql.Open("postgres", connStr) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return p.c.Ping() |
||||
} |
||||
|
||||
// Read returns raw session store by session ID.
|
||||
func (p *PostgresProvider) Read(sid string) (session.RawStore, error) { |
||||
var data []byte |
||||
err := p.c.QueryRow("SELECT data FROM session WHERE key=$1", sid).Scan(&data) |
||||
if err == sql.ErrNoRows { |
||||
_, err = p.c.Exec("INSERT INTO session(key,data,expiry) VALUES($1,$2,$3)", |
||||
sid, "", time.Now().Unix()) |
||||
} |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
var kv map[interface{}]interface{} |
||||
if len(data) == 0 { |
||||
kv = make(map[interface{}]interface{}) |
||||
} else { |
||||
kv, err = session.DecodeGob(data) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
return NewPostgresStore(p.c, sid, kv), nil |
||||
} |
||||
|
||||
// Exist returns true if session with given ID exists.
|
||||
func (p *PostgresProvider) Exist(sid string) bool { |
||||
var data []byte |
||||
err := p.c.QueryRow("SELECT data FROM session WHERE key=$1", sid).Scan(&data) |
||||
if err != nil && err != sql.ErrNoRows { |
||||
panic("session/postgres: error checking existence: " + err.Error()) |
||||
} |
||||
return err != sql.ErrNoRows |
||||
} |
||||
|
||||
// Destory deletes a session by session ID.
|
||||
func (p *PostgresProvider) Destory(sid string) error { |
||||
_, err := p.c.Exec("DELETE FROM session WHERE key=$1", sid) |
||||
return err |
||||
} |
||||
|
||||
// Regenerate regenerates a session store from old session ID to new one.
|
||||
func (p *PostgresProvider) Regenerate(oldsid, sid string) (_ session.RawStore, err error) { |
||||
if p.Exist(sid) { |
||||
return nil, fmt.Errorf("new sid '%s' already exists", sid) |
||||
} |
||||
|
||||
if !p.Exist(oldsid) { |
||||
if _, err = p.c.Exec("INSERT INTO session(key,data,expiry) VALUES($1,$2,$3)", |
||||
oldsid, "", time.Now().Unix()); err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
if _, err = p.c.Exec("UPDATE session SET key=$1 WHERE key=$2", sid, oldsid); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return p.Read(sid) |
||||
} |
||||
|
||||
// Count counts and returns number of sessions.
|
||||
func (p *PostgresProvider) Count() (total int) { |
||||
if err := p.c.QueryRow("SELECT COUNT(*) AS NUM FROM session").Scan(&total); err != nil { |
||||
panic("session/postgres: error counting records: " + err.Error()) |
||||
} |
||||
return total |
||||
} |
||||
|
||||
// GC calls GC to clean expired sessions.
|
||||
func (p *PostgresProvider) GC() { |
||||
if _, err := p.c.Exec("DELETE FROM session WHERE EXTRACT(EPOCH FROM NOW()) - expiry > $1", p.maxlifetime); err != nil { |
||||
log.Printf("session/postgres: error garbage collecting: %v", err) |
||||
} |
||||
} |
||||
|
||||
func init() { |
||||
session.Register("postgres", &PostgresProvider{}) |
||||
} |
||||
@ -0,0 +1,27 @@ |
||||
Copyright (c) 2014 - 2016 lunny |
||||
All rights reserved. |
||||
|
||||
Redistribution and use in source and binary forms, with or without |
||||
modification, are permitted provided that the following conditions are met: |
||||
|
||||
* Redistributions of source code must retain the above copyright notice, this |
||||
list of conditions and the following disclaimer. |
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice, |
||||
this list of conditions and the following disclaimer in the documentation |
||||
and/or other materials provided with the distribution. |
||||
|
||||
* Neither the name of the {organization} nor the names of its |
||||
contributors may be used to endorse or promote products derived from |
||||
this software without specific prior written permission. |
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE |
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
@ -0,0 +1,36 @@ |
||||
package log |
||||
|
||||
import ( |
||||
"database/sql" |
||||
"time" |
||||
) |
||||
|
||||
type DBWriter struct { |
||||
db *sql.DB |
||||
stmt *sql.Stmt |
||||
content chan []byte |
||||
} |
||||
|
||||
func NewDBWriter(db *sql.DB) (*DBWriter, error) { |
||||
_, err := db.Exec("CREATE TABLE IF NOT EXISTS log (id int, content text, created datetime)") |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
stmt, err := db.Prepare("INSERT INTO log (content, created) values (?, ?)") |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return &DBWriter{db, stmt, make(chan []byte, 1000)}, nil |
||||
} |
||||
|
||||
func (w *DBWriter) Write(p []byte) (n int, err error) { |
||||
_, err = w.stmt.Exec(string(p), time.Now()) |
||||
if err == nil { |
||||
n = len(p) |
||||
} |
||||
return |
||||
} |
||||
|
||||
func (w *DBWriter) Close() { |
||||
w.stmt.Close() |
||||
} |
||||
@ -0,0 +1,112 @@ |
||||
package log |
||||
|
||||
import ( |
||||
"io" |
||||
"os" |
||||
"path/filepath" |
||||
"sync" |
||||
"time" |
||||
) |
||||
|
||||
var _ io.Writer = &Files{} |
||||
|
||||
type ByType int |
||||
|
||||
const ( |
||||
ByDay ByType = iota |
||||
ByHour |
||||
ByMonth |
||||
) |
||||
|
||||
var ( |
||||
formats = map[ByType]string{ |
||||
ByDay: "2006-01-02", |
||||
ByHour: "2006-01-02-15", |
||||
ByMonth: "2006-01", |
||||
} |
||||
) |
||||
|
||||
func SetFileFormat(t ByType, format string) { |
||||
formats[t] = format |
||||
} |
||||
|
||||
func (b ByType) Format() string { |
||||
return formats[b] |
||||
} |
||||
|
||||
type Files struct { |
||||
FileOptions |
||||
f *os.File |
||||
lastFormat string |
||||
lock sync.Mutex |
||||
} |
||||
|
||||
type FileOptions struct { |
||||
Dir string |
||||
ByType ByType |
||||
Loc *time.Location |
||||
} |
||||
|
||||
func prepareFileOption(opts []FileOptions) FileOptions { |
||||
var opt FileOptions |
||||
if len(opts) > 0 { |
||||
opt = opts[0] |
||||
} |
||||
if opt.Dir == "" { |
||||
opt.Dir = "./" |
||||
} |
||||
err := os.MkdirAll(opt.Dir, os.ModePerm) |
||||
if err != nil { |
||||
panic(err.Error()) |
||||
} |
||||
|
||||
if opt.Loc == nil { |
||||
opt.Loc = time.Local |
||||
} |
||||
return opt |
||||
} |
||||
|
||||
func NewFileWriter(opts ...FileOptions) *Files { |
||||
opt := prepareFileOption(opts) |
||||
return &Files{ |
||||
FileOptions: opt, |
||||
} |
||||
} |
||||
|
||||
func (f *Files) getFile() (*os.File, error) { |
||||
var err error |
||||
t := time.Now().In(f.Loc) |
||||
if f.f == nil { |
||||
f.lastFormat = t.Format(f.ByType.Format()) |
||||
f.f, err = os.OpenFile(filepath.Join(f.Dir, f.lastFormat+".log"), |
||||
os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) |
||||
return f.f, err |
||||
} |
||||
if f.lastFormat != t.Format(f.ByType.Format()) { |
||||
f.f.Close() |
||||
f.lastFormat = t.Format(f.ByType.Format()) |
||||
f.f, err = os.OpenFile(filepath.Join(f.Dir, f.lastFormat+".log"), |
||||
os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) |
||||
return f.f, err |
||||
} |
||||
return f.f, nil |
||||
} |
||||
|
||||
func (f *Files) Write(bs []byte) (int, error) { |
||||
f.lock.Lock() |
||||
defer f.lock.Unlock() |
||||
|
||||
w, err := f.getFile() |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
return w.Write(bs) |
||||
} |
||||
|
||||
func (f *Files) Close() { |
||||
if f.f != nil { |
||||
f.f.Close() |
||||
f.f = nil |
||||
} |
||||
f.lastFormat = "" |
||||
} |
||||
@ -0,0 +1,595 @@ |
||||
package log |
||||
|
||||
import ( |
||||
"bytes" |
||||
"fmt" |
||||
"io" |
||||
"os" |
||||
"runtime" |
||||
"strings" |
||||
"sync" |
||||
"time" |
||||
) |
||||
|
||||
// These flags define which text to prefix to each log entry generated by the Logger.
|
||||
const ( |
||||
// Bits or'ed together to control what's printed. There is no control over the
|
||||
// order they appear (the order listed here) or the format they present (as
|
||||
// described in the comments). A colon appears after these items:
|
||||
// 2009/0123 01:23:23.123123 /a/b/c/d.go:23: message
|
||||
Ldate = 1 << iota // the date: 2009/0123
|
||||
Ltime // the time: 01:23:23
|
||||
Lmicroseconds // microsecond resolution: 01:23:23.123123. assumes Ltime.
|
||||
Llongfile // full file name and line number: /a/b/c/d.go:23
|
||||
Lshortfile // final file name element and line number: d.go:23. overrides Llongfile
|
||||
Lmodule // module name
|
||||
Llevel // level: 0(Debug), 1(Info), 2(Warn), 3(Error), 4(Panic), 5(Fatal)
|
||||
Llongcolor // color will start [info] end of line
|
||||
Lshortcolor // color only include [info]
|
||||
LstdFlags = Ldate | Ltime // initial values for the standard logger
|
||||
//Ldefault = Llevel | LstdFlags | Lshortfile | Llongcolor
|
||||
) // [prefix][time][level][module][shortfile|longfile]
|
||||
|
||||
func Ldefault() int { |
||||
if runtime.GOOS == "windows" { |
||||
return Llevel | LstdFlags | Lshortfile |
||||
} |
||||
return Llevel | LstdFlags | Lshortfile | Llongcolor |
||||
} |
||||
|
||||
func Version() string { |
||||
return "0.2.0.1121" |
||||
} |
||||
|
||||
const ( |
||||
Lall = iota |
||||
) |
||||
const ( |
||||
Ldebug = iota |
||||
Linfo |
||||
Lwarn |
||||
Lerror |
||||
Lpanic |
||||
Lfatal |
||||
Lnone |
||||
) |
||||
|
||||
const ( |
||||
ForeBlack = iota + 30 //30
|
||||
ForeRed //31
|
||||
ForeGreen //32
|
||||
ForeYellow //33
|
||||
ForeBlue //34
|
||||
ForePurple //35
|
||||
ForeCyan //36
|
||||
ForeWhite //37
|
||||
) |
||||
|
||||
const ( |
||||
BackBlack = iota + 40 //40
|
||||
BackRed //41
|
||||
BackGreen //42
|
||||
BackYellow //43
|
||||
BackBlue //44
|
||||
BackPurple //45
|
||||
BackCyan //46
|
||||
BackWhite //47
|
||||
) |
||||
|
||||
var levels = []string{ |
||||
"[Debug]", |
||||
"[Info]", |
||||
"[Warn]", |
||||
"[Error]", |
||||
"[Panic]", |
||||
"[Fatal]", |
||||
} |
||||
|
||||
// MUST called before all logs
|
||||
func SetLevels(lvs []string) { |
||||
levels = lvs |
||||
} |
||||
|
||||
var colors = []int{ |
||||
ForeCyan, |
||||
ForeGreen, |
||||
ForeYellow, |
||||
ForeRed, |
||||
ForePurple, |
||||
ForeBlue, |
||||
} |
||||
|
||||
// MUST called before all logs
|
||||
func SetColors(cls []int) { |
||||
colors = cls |
||||
} |
||||
|
||||
// A Logger represents an active logging object that generates lines of
|
||||
// output to an io.Writer. Each logging operation makes a single call to
|
||||
// the Writer's Write method. A Logger can be used simultaneously from
|
||||
// multiple goroutines; it guarantees to serialize access to the Writer.
|
||||
type Logger struct { |
||||
mu sync.Mutex // ensures atomic writes; protects the following fields
|
||||
prefix string // prefix to write at beginning of each line
|
||||
flag int // properties
|
||||
Level int |
||||
out io.Writer // destination for output
|
||||
buf bytes.Buffer // for accumulating text to write
|
||||
levelStats [6]int64 |
||||
loc *time.Location |
||||
} |
||||
|
||||
// New creates a new Logger. The out variable sets the
|
||||
// destination to which log data will be written.
|
||||
// The prefix appears at the beginning of each generated log line.
|
||||
// The flag argument defines the logging properties.
|
||||
func New(out io.Writer, prefix string, flag int) *Logger { |
||||
l := &Logger{out: out, prefix: prefix, Level: 1, flag: flag, loc: time.Local} |
||||
if out != os.Stdout { |
||||
l.flag = RmColorFlags(l.flag) |
||||
} |
||||
return l |
||||
} |
||||
|
||||
var Std = New(os.Stderr, "", Ldefault()) |
||||
|
||||
// Cheap integer to fixed-width decimal ASCII. Give a negative width to avoid zero-padding.
|
||||
// Knows the buffer has capacity.
|
||||
func itoa(buf *bytes.Buffer, i int, wid int) { |
||||
var u uint = uint(i) |
||||
if u == 0 && wid <= 1 { |
||||
buf.WriteByte('0') |
||||
return |
||||
} |
||||
|
||||
// Assemble decimal in reverse order.
|
||||
var b [32]byte |
||||
bp := len(b) |
||||
for ; u > 0 || wid > 0; u /= 10 { |
||||
bp-- |
||||
wid-- |
||||
b[bp] = byte(u%10) + '0' |
||||
} |
||||
|
||||
// avoid slicing b to avoid an allocation.
|
||||
for bp < len(b) { |
||||
buf.WriteByte(b[bp]) |
||||
bp++ |
||||
} |
||||
} |
||||
|
||||
func moduleOf(file string) string { |
||||
pos := strings.LastIndex(file, "/") |
||||
if pos != -1 { |
||||
pos1 := strings.LastIndex(file[:pos], "/src/") |
||||
if pos1 != -1 { |
||||
return file[pos1+5 : pos] |
||||
} |
||||
} |
||||
return "UNKNOWN" |
||||
} |
||||
|
||||
func (l *Logger) formatHeader(buf *bytes.Buffer, t time.Time, |
||||
file string, line int, lvl int, reqId string) { |
||||
if l.prefix != "" { |
||||
buf.WriteString(l.prefix) |
||||
} |
||||
if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 { |
||||
if l.flag&Ldate != 0 { |
||||
year, month, day := t.Date() |
||||
itoa(buf, year, 4) |
||||
buf.WriteByte('/') |
||||
itoa(buf, int(month), 2) |
||||
buf.WriteByte('/') |
||||
itoa(buf, day, 2) |
||||
buf.WriteByte(' ') |
||||
} |
||||
if l.flag&(Ltime|Lmicroseconds) != 0 { |
||||
hour, min, sec := t.Clock() |
||||
itoa(buf, hour, 2) |
||||
buf.WriteByte(':') |
||||
itoa(buf, min, 2) |
||||
buf.WriteByte(':') |
||||
itoa(buf, sec, 2) |
||||
if l.flag&Lmicroseconds != 0 { |
||||
buf.WriteByte('.') |
||||
itoa(buf, t.Nanosecond()/1e3, 6) |
||||
} |
||||
buf.WriteByte(' ') |
||||
} |
||||
} |
||||
if reqId != "" { |
||||
buf.WriteByte('[') |
||||
buf.WriteString(reqId) |
||||
buf.WriteByte(']') |
||||
buf.WriteByte(' ') |
||||
} |
||||
|
||||
if l.flag&(Lshortcolor|Llongcolor) != 0 { |
||||
buf.WriteString(fmt.Sprintf("\033[1;%dm", colors[lvl])) |
||||
} |
||||
if l.flag&Llevel != 0 { |
||||
buf.WriteString(levels[lvl]) |
||||
buf.WriteByte(' ') |
||||
} |
||||
if l.flag&Lshortcolor != 0 { |
||||
buf.WriteString("\033[0m") |
||||
} |
||||
|
||||
if l.flag&Lmodule != 0 { |
||||
buf.WriteByte('[') |
||||
buf.WriteString(moduleOf(file)) |
||||
buf.WriteByte(']') |
||||
buf.WriteByte(' ') |
||||
} |
||||
if l.flag&(Lshortfile|Llongfile) != 0 { |
||||
if l.flag&Lshortfile != 0 { |
||||
short := file |
||||
for i := len(file) - 1; i > 0; i-- { |
||||
if file[i] == '/' { |
||||
short = file[i+1:] |
||||
break |
||||
} |
||||
} |
||||
file = short |
||||
} |
||||
buf.WriteString(file) |
||||
buf.WriteByte(':') |
||||
itoa(buf, line, -1) |
||||
buf.WriteByte(' ') |
||||
} |
||||
} |
||||
|
||||
// Output writes the output for a logging event. The string s contains
|
||||
// the text to print after the prefix specified by the flags of the
|
||||
// Logger. A newline is appended if the last character of s is not
|
||||
// already a newline. Calldepth is used to recover the PC and is
|
||||
// provided for generality, although at the moment on all pre-defined
|
||||
// paths it will be 2.
|
||||
func (l *Logger) Output(reqId string, lvl int, calldepth int, s string) error { |
||||
if lvl < l.Level { |
||||
return nil |
||||
} |
||||
now := time.Now().In(l.loc) // get this early.
|
||||
var file string |
||||
var line int |
||||
l.mu.Lock() |
||||
defer l.mu.Unlock() |
||||
if l.flag&(Lshortfile|Llongfile|Lmodule) != 0 { |
||||
// release lock while getting caller info - it's expensive.
|
||||
l.mu.Unlock() |
||||
var ok bool |
||||
_, file, line, ok = runtime.Caller(calldepth) |
||||
if !ok { |
||||
file = "???" |
||||
line = 0 |
||||
} |
||||
l.mu.Lock() |
||||
} |
||||
l.levelStats[lvl]++ |
||||
l.buf.Reset() |
||||
l.formatHeader(&l.buf, now, file, line, lvl, reqId) |
||||
l.buf.WriteString(s) |
||||
if l.flag&Llongcolor != 0 { |
||||
l.buf.WriteString("\033[0m") |
||||
} |
||||
if len(s) > 0 && s[len(s)-1] != '\n' { |
||||
l.buf.WriteByte('\n') |
||||
} |
||||
_, err := l.out.Write(l.buf.Bytes()) |
||||
return err |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
|
||||
// Printf calls l.Output to print to the logger.
|
||||
// Arguments are handled in the manner of fmt.Printf.
|
||||
func (l *Logger) Printf(format string, v ...interface{}) { |
||||
l.Output("", Linfo, 2, fmt.Sprintf(format, v...)) |
||||
} |
||||
|
||||
// Print calls l.Output to print to the logger.
|
||||
// Arguments are handled in the manner of fmt.Print.
|
||||
func (l *Logger) Print(v ...interface{}) { |
||||
l.Output("", Linfo, 2, fmt.Sprint(v...)) |
||||
} |
||||
|
||||
// Println calls l.Output to print to the logger.
|
||||
// Arguments are handled in the manner of fmt.Println.
|
||||
func (l *Logger) Println(v ...interface{}) { |
||||
l.Output("", Linfo, 2, fmt.Sprintln(v...)) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
|
||||
func (l *Logger) Debugf(format string, v ...interface{}) { |
||||
l.Output("", Ldebug, 2, fmt.Sprintf(format, v...)) |
||||
} |
||||
|
||||
func (l *Logger) Debug(v ...interface{}) { |
||||
l.Output("", Ldebug, 2, fmt.Sprintln(v...)) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
func (l *Logger) Infof(format string, v ...interface{}) { |
||||
l.Output("", Linfo, 2, fmt.Sprintf(format, v...)) |
||||
} |
||||
|
||||
func (l *Logger) Info(v ...interface{}) { |
||||
l.Output("", Linfo, 2, fmt.Sprintln(v...)) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
func (l *Logger) Warnf(format string, v ...interface{}) { |
||||
l.Output("", Lwarn, 2, fmt.Sprintf(format, v...)) |
||||
} |
||||
|
||||
func (l *Logger) Warn(v ...interface{}) { |
||||
l.Output("", Lwarn, 2, fmt.Sprintln(v...)) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
|
||||
func (l *Logger) Errorf(format string, v ...interface{}) { |
||||
l.Output("", Lerror, 2, fmt.Sprintf(format, v...)) |
||||
} |
||||
|
||||
func (l *Logger) Error(v ...interface{}) { |
||||
l.Output("", Lerror, 2, fmt.Sprintln(v...)) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
|
||||
func (l *Logger) Fatal(v ...interface{}) { |
||||
l.Output("", Lfatal, 2, fmt.Sprintln(v...)) |
||||
os.Exit(1) |
||||
} |
||||
|
||||
// Fatalf is equivalent to l.Printf() followed by a call to os.Exit(1).
|
||||
func (l *Logger) Fatalf(format string, v ...interface{}) { |
||||
l.Output("", Lfatal, 2, fmt.Sprintf(format, v...)) |
||||
os.Exit(1) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
// Panic is equivalent to l.Print() followed by a call to panic().
|
||||
func (l *Logger) Panic(v ...interface{}) { |
||||
s := fmt.Sprintln(v...) |
||||
l.Output("", Lpanic, 2, s) |
||||
panic(s) |
||||
} |
||||
|
||||
// Panicf is equivalent to l.Printf() followed by a call to panic().
|
||||
func (l *Logger) Panicf(format string, v ...interface{}) { |
||||
s := fmt.Sprintf(format, v...) |
||||
l.Output("", Lpanic, 2, s) |
||||
panic(s) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
func (l *Logger) Stack(v ...interface{}) { |
||||
s := fmt.Sprint(v...) |
||||
s += "\n" |
||||
buf := make([]byte, 1024*1024) |
||||
n := runtime.Stack(buf, true) |
||||
s += string(buf[:n]) |
||||
s += "\n" |
||||
l.Output("", Lerror, 2, s) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
func (l *Logger) Stat() (stats []int64) { |
||||
l.mu.Lock() |
||||
v := l.levelStats |
||||
l.mu.Unlock() |
||||
return v[:] |
||||
} |
||||
|
||||
// Flags returns the output flags for the logger.
|
||||
func (l *Logger) Flags() int { |
||||
l.mu.Lock() |
||||
defer l.mu.Unlock() |
||||
return l.flag |
||||
} |
||||
|
||||
func RmColorFlags(flag int) int { |
||||
// for un std out, it should not show color since almost them don't support
|
||||
if flag&Llongcolor != 0 { |
||||
flag = flag ^ Llongcolor |
||||
} |
||||
if flag&Lshortcolor != 0 { |
||||
flag = flag ^ Lshortcolor |
||||
} |
||||
return flag |
||||
} |
||||
|
||||
func (l *Logger) Location() *time.Location { |
||||
return l.loc |
||||
} |
||||
|
||||
func (l *Logger) SetLocation(loc *time.Location) { |
||||
l.loc = loc |
||||
} |
||||
|
||||
// SetFlags sets the output flags for the logger.
|
||||
func (l *Logger) SetFlags(flag int) { |
||||
l.mu.Lock() |
||||
defer l.mu.Unlock() |
||||
if l.out != os.Stdout { |
||||
flag = RmColorFlags(flag) |
||||
} |
||||
l.flag = flag |
||||
} |
||||
|
||||
// Prefix returns the output prefix for the logger.
|
||||
func (l *Logger) Prefix() string { |
||||
l.mu.Lock() |
||||
defer l.mu.Unlock() |
||||
return l.prefix |
||||
} |
||||
|
||||
// SetPrefix sets the output prefix for the logger.
|
||||
func (l *Logger) SetPrefix(prefix string) { |
||||
l.mu.Lock() |
||||
defer l.mu.Unlock() |
||||
l.prefix = prefix |
||||
} |
||||
|
||||
// SetOutputLevel sets the output level for the logger.
|
||||
func (l *Logger) SetOutputLevel(lvl int) { |
||||
l.mu.Lock() |
||||
defer l.mu.Unlock() |
||||
l.Level = lvl |
||||
} |
||||
|
||||
func (l *Logger) OutputLevel() int { |
||||
return l.Level |
||||
} |
||||
|
||||
func (l *Logger) SetOutput(w io.Writer) { |
||||
l.mu.Lock() |
||||
defer l.mu.Unlock() |
||||
l.out = w |
||||
if w != os.Stdout { |
||||
l.flag = RmColorFlags(l.flag) |
||||
} |
||||
} |
||||
|
||||
// SetOutput sets the output destination for the standard logger.
|
||||
func SetOutput(w io.Writer) { |
||||
Std.SetOutput(w) |
||||
} |
||||
|
||||
func SetLocation(loc *time.Location) { |
||||
Std.SetLocation(loc) |
||||
} |
||||
|
||||
func Location() *time.Location { |
||||
return Std.Location() |
||||
} |
||||
|
||||
// Flags returns the output flags for the standard logger.
|
||||
func Flags() int { |
||||
return Std.Flags() |
||||
} |
||||
|
||||
// SetFlags sets the output flags for the standard logger.
|
||||
func SetFlags(flag int) { |
||||
Std.SetFlags(flag) |
||||
} |
||||
|
||||
// Prefix returns the output prefix for the standard logger.
|
||||
func Prefix() string { |
||||
return Std.Prefix() |
||||
} |
||||
|
||||
// SetPrefix sets the output prefix for the standard logger.
|
||||
func SetPrefix(prefix string) { |
||||
Std.SetPrefix(prefix) |
||||
} |
||||
|
||||
func SetOutputLevel(lvl int) { |
||||
Std.SetOutputLevel(lvl) |
||||
} |
||||
|
||||
func OutputLevel() int { |
||||
return Std.OutputLevel() |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
|
||||
// Print calls Output to print to the standard logger.
|
||||
// Arguments are handled in the manner of fmt.Print.
|
||||
func Print(v ...interface{}) { |
||||
Std.Output("", Linfo, 2, fmt.Sprintln(v...)) |
||||
} |
||||
|
||||
// Printf calls Output to print to the standard logger.
|
||||
// Arguments are handled in the manner of fmt.Printf.
|
||||
func Printf(format string, v ...interface{}) { |
||||
Std.Output("", Linfo, 2, fmt.Sprintf(format, v...)) |
||||
} |
||||
|
||||
// Println calls Output to print to the standard logger.
|
||||
// Arguments are handled in the manner of fmt.Println.
|
||||
func Println(v ...interface{}) { |
||||
Std.Output("", Linfo, 2, fmt.Sprintln(v...)) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
|
||||
func Debugf(format string, v ...interface{}) { |
||||
Std.Output("", Ldebug, 2, fmt.Sprintf(format, v...)) |
||||
} |
||||
|
||||
func Debug(v ...interface{}) { |
||||
Std.Output("", Ldebug, 2, fmt.Sprintln(v...)) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
|
||||
func Infof(format string, v ...interface{}) { |
||||
Std.Output("", Linfo, 2, fmt.Sprintf(format, v...)) |
||||
} |
||||
|
||||
func Info(v ...interface{}) { |
||||
Std.Output("", Linfo, 2, fmt.Sprintln(v...)) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
|
||||
func Warnf(format string, v ...interface{}) { |
||||
Std.Output("", Lwarn, 2, fmt.Sprintf(format, v...)) |
||||
} |
||||
|
||||
func Warn(v ...interface{}) { |
||||
Std.Output("", Lwarn, 2, fmt.Sprintln(v...)) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
|
||||
func Errorf(format string, v ...interface{}) { |
||||
Std.Output("", Lerror, 2, fmt.Sprintf(format, v...)) |
||||
} |
||||
|
||||
func Error(v ...interface{}) { |
||||
Std.Output("", Lerror, 2, fmt.Sprintln(v...)) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
|
||||
// Fatal is equivalent to Print() followed by a call to os.Exit(1).
|
||||
func Fatal(v ...interface{}) { |
||||
Std.Output("", Lfatal, 2, fmt.Sprintln(v...)) |
||||
} |
||||
|
||||
// Fatalf is equivalent to Printf() followed by a call to os.Exit(1).
|
||||
func Fatalf(format string, v ...interface{}) { |
||||
Std.Output("", Lfatal, 2, fmt.Sprintf(format, v...)) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
|
||||
// Panic is equivalent to Print() followed by a call to panic().
|
||||
func Panic(v ...interface{}) { |
||||
Std.Output("", Lpanic, 2, fmt.Sprintln(v...)) |
||||
} |
||||
|
||||
// Panicf is equivalent to Printf() followed by a call to panic().
|
||||
func Panicf(format string, v ...interface{}) { |
||||
Std.Output("", Lpanic, 2, fmt.Sprintf(format, v...)) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
|
||||
func Stack(v ...interface{}) { |
||||
s := fmt.Sprint(v...) |
||||
s += "\n" |
||||
buf := make([]byte, 1024*1024) |
||||
n := runtime.Stack(buf, true) |
||||
s += string(buf[:n]) |
||||
s += "\n" |
||||
Std.Output("", Lerror, 2, s) |
||||
} |
||||
|
||||
// -----------------------------------------
|
||||
@ -0,0 +1,21 @@ |
||||
The MIT License (MIT) |
||||
|
||||
Copyright (c) 2014 siddontang |
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||
of this software and associated documentation files (the "Software"), to deal |
||||
in the Software without restriction, including without limitation the rights |
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||
copies of the Software, and to permit persons to whom the Software is |
||||
furnished to do so, subject to the following conditions: |
||||
|
||||
The above copyright notice and this permission notice shall be included in all |
||||
copies or substantial portions of the Software. |
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
||||
SOFTWARE. |
||||
@ -0,0 +1,106 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"sync" |
||||
|
||||
"github.com/lunny/nodb/store" |
||||
) |
||||
|
||||
type batch struct { |
||||
l *Nodb |
||||
|
||||
store.WriteBatch |
||||
|
||||
sync.Locker |
||||
|
||||
logs [][]byte |
||||
|
||||
tx *Tx |
||||
} |
||||
|
||||
func (b *batch) Commit() error { |
||||
b.l.commitLock.Lock() |
||||
defer b.l.commitLock.Unlock() |
||||
|
||||
err := b.WriteBatch.Commit() |
||||
|
||||
if b.l.binlog != nil { |
||||
if err == nil { |
||||
if b.tx == nil { |
||||
b.l.binlog.Log(b.logs...) |
||||
} else { |
||||
b.tx.logs = append(b.tx.logs, b.logs...) |
||||
} |
||||
} |
||||
b.logs = [][]byte{} |
||||
} |
||||
|
||||
return err |
||||
} |
||||
|
||||
func (b *batch) Lock() { |
||||
b.Locker.Lock() |
||||
} |
||||
|
||||
func (b *batch) Unlock() { |
||||
if b.l.binlog != nil { |
||||
b.logs = [][]byte{} |
||||
} |
||||
b.WriteBatch.Rollback() |
||||
b.Locker.Unlock() |
||||
} |
||||
|
||||
func (b *batch) Put(key []byte, value []byte) { |
||||
if b.l.binlog != nil { |
||||
buf := encodeBinLogPut(key, value) |
||||
b.logs = append(b.logs, buf) |
||||
} |
||||
b.WriteBatch.Put(key, value) |
||||
} |
||||
|
||||
func (b *batch) Delete(key []byte) { |
||||
if b.l.binlog != nil { |
||||
buf := encodeBinLogDelete(key) |
||||
b.logs = append(b.logs, buf) |
||||
} |
||||
b.WriteBatch.Delete(key) |
||||
} |
||||
|
||||
type dbBatchLocker struct { |
||||
l *sync.Mutex |
||||
wrLock *sync.RWMutex |
||||
} |
||||
|
||||
func (l *dbBatchLocker) Lock() { |
||||
l.wrLock.RLock() |
||||
l.l.Lock() |
||||
} |
||||
|
||||
func (l *dbBatchLocker) Unlock() { |
||||
l.l.Unlock() |
||||
l.wrLock.RUnlock() |
||||
} |
||||
|
||||
type txBatchLocker struct { |
||||
} |
||||
|
||||
func (l *txBatchLocker) Lock() {} |
||||
func (l *txBatchLocker) Unlock() {} |
||||
|
||||
type multiBatchLocker struct { |
||||
} |
||||
|
||||
func (l *multiBatchLocker) Lock() {} |
||||
func (l *multiBatchLocker) Unlock() {} |
||||
|
||||
func (l *Nodb) newBatch(wb store.WriteBatch, locker sync.Locker, tx *Tx) *batch { |
||||
b := new(batch) |
||||
b.l = l |
||||
b.WriteBatch = wb |
||||
|
||||
b.tx = tx |
||||
b.Locker = locker |
||||
|
||||
b.logs = [][]byte{} |
||||
return b |
||||
} |
||||
@ -0,0 +1,391 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"bufio" |
||||
"encoding/binary" |
||||
"fmt" |
||||
"io" |
||||
"io/ioutil" |
||||
"os" |
||||
"path" |
||||
"strconv" |
||||
"strings" |
||||
"sync" |
||||
"time" |
||||
|
||||
"github.com/lunny/log" |
||||
"github.com/lunny/nodb/config" |
||||
) |
||||
|
||||
type BinLogHead struct { |
||||
CreateTime uint32 |
||||
BatchId uint32 |
||||
PayloadLen uint32 |
||||
} |
||||
|
||||
func (h *BinLogHead) Len() int { |
||||
return 12 |
||||
} |
||||
|
||||
func (h *BinLogHead) Write(w io.Writer) error { |
||||
if err := binary.Write(w, binary.BigEndian, h.CreateTime); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if err := binary.Write(w, binary.BigEndian, h.BatchId); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if err := binary.Write(w, binary.BigEndian, h.PayloadLen); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (h *BinLogHead) handleReadError(err error) error { |
||||
if err == io.EOF { |
||||
return io.ErrUnexpectedEOF |
||||
} else { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
func (h *BinLogHead) Read(r io.Reader) error { |
||||
var err error |
||||
if err = binary.Read(r, binary.BigEndian, &h.CreateTime); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if err = binary.Read(r, binary.BigEndian, &h.BatchId); err != nil { |
||||
return h.handleReadError(err) |
||||
} |
||||
|
||||
if err = binary.Read(r, binary.BigEndian, &h.PayloadLen); err != nil { |
||||
return h.handleReadError(err) |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (h *BinLogHead) InSameBatch(ho *BinLogHead) bool { |
||||
if h.CreateTime == ho.CreateTime && h.BatchId == ho.BatchId { |
||||
return true |
||||
} else { |
||||
return false |
||||
} |
||||
} |
||||
|
||||
/* |
||||
index file format: |
||||
ledis-bin.00001 |
||||
ledis-bin.00002 |
||||
ledis-bin.00003 |
||||
|
||||
log file format |
||||
|
||||
Log: Head|PayloadData |
||||
|
||||
Head: createTime|batchId|payloadData |
||||
|
||||
*/ |
||||
|
||||
type BinLog struct { |
||||
sync.Mutex |
||||
|
||||
path string |
||||
|
||||
cfg *config.BinLogConfig |
||||
|
||||
logFile *os.File |
||||
|
||||
logWb *bufio.Writer |
||||
|
||||
indexName string |
||||
logNames []string |
||||
lastLogIndex int64 |
||||
|
||||
batchId uint32 |
||||
|
||||
ch chan struct{} |
||||
} |
||||
|
||||
func NewBinLog(cfg *config.Config) (*BinLog, error) { |
||||
l := new(BinLog) |
||||
|
||||
l.cfg = &cfg.BinLog |
||||
l.cfg.Adjust() |
||||
|
||||
l.path = path.Join(cfg.DataDir, "binlog") |
||||
|
||||
if err := os.MkdirAll(l.path, os.ModePerm); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
l.logNames = make([]string, 0, 16) |
||||
|
||||
l.ch = make(chan struct{}) |
||||
|
||||
if err := l.loadIndex(); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return l, nil |
||||
} |
||||
|
||||
func (l *BinLog) flushIndex() error { |
||||
data := strings.Join(l.logNames, "\n") |
||||
|
||||
bakName := fmt.Sprintf("%s.bak", l.indexName) |
||||
f, err := os.OpenFile(bakName, os.O_WRONLY|os.O_CREATE, 0666) |
||||
if err != nil { |
||||
log.Error("create binlog bak index error %s", err.Error()) |
||||
return err |
||||
} |
||||
|
||||
if _, err := f.WriteString(data); err != nil { |
||||
log.Error("write binlog index error %s", err.Error()) |
||||
f.Close() |
||||
return err |
||||
} |
||||
|
||||
f.Close() |
||||
|
||||
if err := os.Rename(bakName, l.indexName); err != nil { |
||||
log.Error("rename binlog bak index error %s", err.Error()) |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (l *BinLog) loadIndex() error { |
||||
l.indexName = path.Join(l.path, fmt.Sprintf("ledis-bin.index")) |
||||
if _, err := os.Stat(l.indexName); os.IsNotExist(err) { |
||||
//no index file, nothing to do
|
||||
} else { |
||||
indexData, err := ioutil.ReadFile(l.indexName) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
lines := strings.Split(string(indexData), "\n") |
||||
for _, line := range lines { |
||||
line = strings.Trim(line, "\r\n ") |
||||
if len(line) == 0 { |
||||
continue |
||||
} |
||||
|
||||
if _, err := os.Stat(path.Join(l.path, line)); err != nil { |
||||
log.Error("load index line %s error %s", line, err.Error()) |
||||
return err |
||||
} else { |
||||
l.logNames = append(l.logNames, line) |
||||
} |
||||
} |
||||
} |
||||
if l.cfg.MaxFileNum > 0 && len(l.logNames) > l.cfg.MaxFileNum { |
||||
//remove oldest logfile
|
||||
if err := l.Purge(len(l.logNames) - l.cfg.MaxFileNum); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
var err error |
||||
if len(l.logNames) == 0 { |
||||
l.lastLogIndex = 1 |
||||
} else { |
||||
lastName := l.logNames[len(l.logNames)-1] |
||||
|
||||
if l.lastLogIndex, err = strconv.ParseInt(path.Ext(lastName)[1:], 10, 64); err != nil { |
||||
log.Error("invalid logfile name %s", err.Error()) |
||||
return err |
||||
} |
||||
|
||||
//like mysql, if server restart, a new binlog will create
|
||||
l.lastLogIndex++ |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (l *BinLog) getLogFile() string { |
||||
return l.FormatLogFileName(l.lastLogIndex) |
||||
} |
||||
|
||||
func (l *BinLog) openNewLogFile() error { |
||||
var err error |
||||
lastName := l.getLogFile() |
||||
|
||||
logPath := path.Join(l.path, lastName) |
||||
if l.logFile, err = os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY, 0666); err != nil { |
||||
log.Error("open new logfile error %s", err.Error()) |
||||
return err |
||||
} |
||||
|
||||
if l.cfg.MaxFileNum > 0 && len(l.logNames) == l.cfg.MaxFileNum { |
||||
l.purge(1) |
||||
} |
||||
|
||||
l.logNames = append(l.logNames, lastName) |
||||
|
||||
if l.logWb == nil { |
||||
l.logWb = bufio.NewWriterSize(l.logFile, 1024) |
||||
} else { |
||||
l.logWb.Reset(l.logFile) |
||||
} |
||||
|
||||
if err = l.flushIndex(); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (l *BinLog) checkLogFileSize() bool { |
||||
if l.logFile == nil { |
||||
return false |
||||
} |
||||
|
||||
st, _ := l.logFile.Stat() |
||||
if st.Size() >= int64(l.cfg.MaxFileSize) { |
||||
l.closeLog() |
||||
return true |
||||
} |
||||
|
||||
return false |
||||
} |
||||
|
||||
func (l *BinLog) closeLog() { |
||||
l.lastLogIndex++ |
||||
|
||||
l.logFile.Close() |
||||
l.logFile = nil |
||||
} |
||||
|
||||
func (l *BinLog) purge(n int) { |
||||
for i := 0; i < n; i++ { |
||||
logPath := path.Join(l.path, l.logNames[i]) |
||||
os.Remove(logPath) |
||||
} |
||||
|
||||
copy(l.logNames[0:], l.logNames[n:]) |
||||
l.logNames = l.logNames[0 : len(l.logNames)-n] |
||||
} |
||||
|
||||
func (l *BinLog) Close() { |
||||
if l.logFile != nil { |
||||
l.logFile.Close() |
||||
l.logFile = nil |
||||
} |
||||
} |
||||
|
||||
func (l *BinLog) LogNames() []string { |
||||
return l.logNames |
||||
} |
||||
|
||||
func (l *BinLog) LogFileName() string { |
||||
return l.getLogFile() |
||||
} |
||||
|
||||
func (l *BinLog) LogFilePos() int64 { |
||||
if l.logFile == nil { |
||||
return 0 |
||||
} else { |
||||
st, _ := l.logFile.Stat() |
||||
return st.Size() |
||||
} |
||||
} |
||||
|
||||
func (l *BinLog) LogFileIndex() int64 { |
||||
return l.lastLogIndex |
||||
} |
||||
|
||||
func (l *BinLog) FormatLogFileName(index int64) string { |
||||
return fmt.Sprintf("ledis-bin.%07d", index) |
||||
} |
||||
|
||||
func (l *BinLog) FormatLogFilePath(index int64) string { |
||||
return path.Join(l.path, l.FormatLogFileName(index)) |
||||
} |
||||
|
||||
func (l *BinLog) LogPath() string { |
||||
return l.path |
||||
} |
||||
|
||||
func (l *BinLog) Purge(n int) error { |
||||
l.Lock() |
||||
defer l.Unlock() |
||||
|
||||
if len(l.logNames) == 0 { |
||||
return nil |
||||
} |
||||
|
||||
if n >= len(l.logNames) { |
||||
n = len(l.logNames) |
||||
//can not purge current log file
|
||||
if l.logNames[n-1] == l.getLogFile() { |
||||
n = n - 1 |
||||
} |
||||
} |
||||
|
||||
l.purge(n) |
||||
|
||||
return l.flushIndex() |
||||
} |
||||
|
||||
func (l *BinLog) PurgeAll() error { |
||||
l.Lock() |
||||
defer l.Unlock() |
||||
|
||||
l.closeLog() |
||||
return l.openNewLogFile() |
||||
} |
||||
|
||||
func (l *BinLog) Log(args ...[]byte) error { |
||||
l.Lock() |
||||
defer l.Unlock() |
||||
|
||||
var err error |
||||
|
||||
if l.logFile == nil { |
||||
if err = l.openNewLogFile(); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
head := &BinLogHead{} |
||||
|
||||
head.CreateTime = uint32(time.Now().Unix()) |
||||
head.BatchId = l.batchId |
||||
|
||||
l.batchId++ |
||||
|
||||
for _, data := range args { |
||||
head.PayloadLen = uint32(len(data)) |
||||
|
||||
if err := head.Write(l.logWb); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if _, err := l.logWb.Write(data); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
if err = l.logWb.Flush(); err != nil { |
||||
log.Error("write log error %s", err.Error()) |
||||
return err |
||||
} |
||||
|
||||
l.checkLogFileSize() |
||||
|
||||
close(l.ch) |
||||
l.ch = make(chan struct{}) |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (l *BinLog) Wait() <-chan struct{} { |
||||
return l.ch |
||||
} |
||||
@ -0,0 +1,215 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
"errors" |
||||
"fmt" |
||||
"strconv" |
||||
) |
||||
|
||||
var ( |
||||
errBinLogDeleteType = errors.New("invalid bin log delete type") |
||||
errBinLogPutType = errors.New("invalid bin log put type") |
||||
errBinLogCommandType = errors.New("invalid bin log command type") |
||||
) |
||||
|
||||
func encodeBinLogDelete(key []byte) []byte { |
||||
buf := make([]byte, 1+len(key)) |
||||
buf[0] = BinLogTypeDeletion |
||||
copy(buf[1:], key) |
||||
return buf |
||||
} |
||||
|
||||
func decodeBinLogDelete(sz []byte) ([]byte, error) { |
||||
if len(sz) < 1 || sz[0] != BinLogTypeDeletion { |
||||
return nil, errBinLogDeleteType |
||||
} |
||||
|
||||
return sz[1:], nil |
||||
} |
||||
|
||||
func encodeBinLogPut(key []byte, value []byte) []byte { |
||||
buf := make([]byte, 3+len(key)+len(value)) |
||||
buf[0] = BinLogTypePut |
||||
pos := 1 |
||||
binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) |
||||
pos += 2 |
||||
copy(buf[pos:], key) |
||||
pos += len(key) |
||||
copy(buf[pos:], value) |
||||
|
||||
return buf |
||||
} |
||||
|
||||
func decodeBinLogPut(sz []byte) ([]byte, []byte, error) { |
||||
if len(sz) < 3 || sz[0] != BinLogTypePut { |
||||
return nil, nil, errBinLogPutType |
||||
} |
||||
|
||||
keyLen := int(binary.BigEndian.Uint16(sz[1:])) |
||||
if 3+keyLen > len(sz) { |
||||
return nil, nil, errBinLogPutType |
||||
} |
||||
|
||||
return sz[3 : 3+keyLen], sz[3+keyLen:], nil |
||||
} |
||||
|
||||
func FormatBinLogEvent(event []byte) (string, error) { |
||||
logType := uint8(event[0]) |
||||
|
||||
var err error |
||||
var k []byte |
||||
var v []byte |
||||
|
||||
var buf []byte = make([]byte, 0, 1024) |
||||
|
||||
switch logType { |
||||
case BinLogTypePut: |
||||
k, v, err = decodeBinLogPut(event) |
||||
buf = append(buf, "PUT "...) |
||||
case BinLogTypeDeletion: |
||||
k, err = decodeBinLogDelete(event) |
||||
buf = append(buf, "DELETE "...) |
||||
default: |
||||
err = errInvalidBinLogEvent |
||||
} |
||||
|
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
|
||||
if buf, err = formatDataKey(buf, k); err != nil { |
||||
return "", err |
||||
} |
||||
|
||||
if v != nil && len(v) != 0 { |
||||
buf = append(buf, fmt.Sprintf(" %q", v)...) |
||||
} |
||||
|
||||
return String(buf), nil |
||||
} |
||||
|
||||
func formatDataKey(buf []byte, k []byte) ([]byte, error) { |
||||
if len(k) < 2 { |
||||
return nil, errInvalidBinLogEvent |
||||
} |
||||
|
||||
buf = append(buf, fmt.Sprintf("DB:%2d ", k[0])...) |
||||
buf = append(buf, fmt.Sprintf("%s ", TypeName[k[1]])...) |
||||
|
||||
db := new(DB) |
||||
db.index = k[0] |
||||
|
||||
//to do format at respective place
|
||||
|
||||
switch k[1] { |
||||
case KVType: |
||||
if key, err := db.decodeKVKey(k); err != nil { |
||||
return nil, err |
||||
} else { |
||||
buf = strconv.AppendQuote(buf, String(key)) |
||||
} |
||||
case HashType: |
||||
if key, field, err := db.hDecodeHashKey(k); err != nil { |
||||
return nil, err |
||||
} else { |
||||
buf = strconv.AppendQuote(buf, String(key)) |
||||
buf = append(buf, ' ') |
||||
buf = strconv.AppendQuote(buf, String(field)) |
||||
} |
||||
case HSizeType: |
||||
if key, err := db.hDecodeSizeKey(k); err != nil { |
||||
return nil, err |
||||
} else { |
||||
buf = strconv.AppendQuote(buf, String(key)) |
||||
} |
||||
case ListType: |
||||
if key, seq, err := db.lDecodeListKey(k); err != nil { |
||||
return nil, err |
||||
} else { |
||||
buf = strconv.AppendQuote(buf, String(key)) |
||||
buf = append(buf, ' ') |
||||
buf = strconv.AppendInt(buf, int64(seq), 10) |
||||
} |
||||
case LMetaType: |
||||
if key, err := db.lDecodeMetaKey(k); err != nil { |
||||
return nil, err |
||||
} else { |
||||
buf = strconv.AppendQuote(buf, String(key)) |
||||
} |
||||
case ZSetType: |
||||
if key, m, err := db.zDecodeSetKey(k); err != nil { |
||||
return nil, err |
||||
} else { |
||||
buf = strconv.AppendQuote(buf, String(key)) |
||||
buf = append(buf, ' ') |
||||
buf = strconv.AppendQuote(buf, String(m)) |
||||
} |
||||
case ZSizeType: |
||||
if key, err := db.zDecodeSizeKey(k); err != nil { |
||||
return nil, err |
||||
} else { |
||||
buf = strconv.AppendQuote(buf, String(key)) |
||||
} |
||||
case ZScoreType: |
||||
if key, m, score, err := db.zDecodeScoreKey(k); err != nil { |
||||
return nil, err |
||||
} else { |
||||
buf = strconv.AppendQuote(buf, String(key)) |
||||
buf = append(buf, ' ') |
||||
buf = strconv.AppendQuote(buf, String(m)) |
||||
buf = append(buf, ' ') |
||||
buf = strconv.AppendInt(buf, score, 10) |
||||
} |
||||
case BitType: |
||||
if key, seq, err := db.bDecodeBinKey(k); err != nil { |
||||
return nil, err |
||||
} else { |
||||
buf = strconv.AppendQuote(buf, String(key)) |
||||
buf = append(buf, ' ') |
||||
buf = strconv.AppendUint(buf, uint64(seq), 10) |
||||
} |
||||
case BitMetaType: |
||||
if key, err := db.bDecodeMetaKey(k); err != nil { |
||||
return nil, err |
||||
} else { |
||||
buf = strconv.AppendQuote(buf, String(key)) |
||||
} |
||||
case SetType: |
||||
if key, member, err := db.sDecodeSetKey(k); err != nil { |
||||
return nil, err |
||||
} else { |
||||
buf = strconv.AppendQuote(buf, String(key)) |
||||
buf = append(buf, ' ') |
||||
buf = strconv.AppendQuote(buf, String(member)) |
||||
} |
||||
case SSizeType: |
||||
if key, err := db.sDecodeSizeKey(k); err != nil { |
||||
return nil, err |
||||
} else { |
||||
buf = strconv.AppendQuote(buf, String(key)) |
||||
} |
||||
case ExpTimeType: |
||||
if tp, key, t, err := db.expDecodeTimeKey(k); err != nil { |
||||
return nil, err |
||||
} else { |
||||
buf = append(buf, TypeName[tp]...) |
||||
buf = append(buf, ' ') |
||||
buf = strconv.AppendQuote(buf, String(key)) |
||||
buf = append(buf, ' ') |
||||
buf = strconv.AppendInt(buf, t, 10) |
||||
} |
||||
case ExpMetaType: |
||||
if tp, key, err := db.expDecodeMetaKey(k); err != nil { |
||||
return nil, err |
||||
} else { |
||||
buf = append(buf, TypeName[tp]...) |
||||
buf = append(buf, ' ') |
||||
buf = strconv.AppendQuote(buf, String(key)) |
||||
} |
||||
default: |
||||
return nil, errInvalidBinLogEvent |
||||
} |
||||
|
||||
return buf, nil |
||||
} |
||||
@ -0,0 +1,135 @@ |
||||
package config |
||||
|
||||
import ( |
||||
"io/ioutil" |
||||
|
||||
"github.com/BurntSushi/toml" |
||||
) |
||||
|
||||
type Size int |
||||
|
||||
const ( |
||||
DefaultAddr string = "127.0.0.1:6380" |
||||
DefaultHttpAddr string = "127.0.0.1:11181" |
||||
|
||||
DefaultDBName string = "goleveldb" |
||||
|
||||
DefaultDataDir string = "./data" |
||||
) |
||||
|
||||
const ( |
||||
MaxBinLogFileSize int = 1024 * 1024 * 1024 |
||||
MaxBinLogFileNum int = 10000 |
||||
|
||||
DefaultBinLogFileSize int = MaxBinLogFileSize |
||||
DefaultBinLogFileNum int = 10 |
||||
) |
||||
|
||||
type LevelDBConfig struct { |
||||
Compression bool `toml:"compression"` |
||||
BlockSize int `toml:"block_size"` |
||||
WriteBufferSize int `toml:"write_buffer_size"` |
||||
CacheSize int `toml:"cache_size"` |
||||
MaxOpenFiles int `toml:"max_open_files"` |
||||
} |
||||
|
||||
type LMDBConfig struct { |
||||
MapSize int `toml:"map_size"` |
||||
NoSync bool `toml:"nosync"` |
||||
} |
||||
|
||||
type BinLogConfig struct { |
||||
MaxFileSize int `toml:"max_file_size"` |
||||
MaxFileNum int `toml:"max_file_num"` |
||||
} |
||||
|
||||
type Config struct { |
||||
DataDir string `toml:"data_dir"` |
||||
|
||||
DBName string `toml:"db_name"` |
||||
|
||||
LevelDB LevelDBConfig `toml:"leveldb"` |
||||
|
||||
LMDB LMDBConfig `toml:"lmdb"` |
||||
|
||||
BinLog BinLogConfig `toml:"binlog"` |
||||
|
||||
SlaveOf string `toml:"slaveof"` |
||||
|
||||
AccessLog string `toml:"access_log"` |
||||
} |
||||
|
||||
func NewConfigWithFile(fileName string) (*Config, error) { |
||||
data, err := ioutil.ReadFile(fileName) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return NewConfigWithData(data) |
||||
} |
||||
|
||||
func NewConfigWithData(data []byte) (*Config, error) { |
||||
cfg := NewConfigDefault() |
||||
|
||||
_, err := toml.Decode(string(data), cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return cfg, nil |
||||
} |
||||
|
||||
func NewConfigDefault() *Config { |
||||
cfg := new(Config) |
||||
|
||||
cfg.DataDir = DefaultDataDir |
||||
|
||||
cfg.DBName = DefaultDBName |
||||
|
||||
// disable binlog
|
||||
cfg.BinLog.MaxFileNum = 0 |
||||
cfg.BinLog.MaxFileSize = 0 |
||||
|
||||
// disable replication
|
||||
cfg.SlaveOf = "" |
||||
|
||||
// disable access log
|
||||
cfg.AccessLog = "" |
||||
|
||||
cfg.LMDB.MapSize = 20 * 1024 * 1024 |
||||
cfg.LMDB.NoSync = true |
||||
|
||||
return cfg |
||||
} |
||||
|
||||
func (cfg *LevelDBConfig) Adjust() { |
||||
if cfg.CacheSize <= 0 { |
||||
cfg.CacheSize = 4 * 1024 * 1024 |
||||
} |
||||
|
||||
if cfg.BlockSize <= 0 { |
||||
cfg.BlockSize = 4 * 1024 |
||||
} |
||||
|
||||
if cfg.WriteBufferSize <= 0 { |
||||
cfg.WriteBufferSize = 4 * 1024 * 1024 |
||||
} |
||||
|
||||
if cfg.MaxOpenFiles < 1024 { |
||||
cfg.MaxOpenFiles = 1024 |
||||
} |
||||
} |
||||
|
||||
func (cfg *BinLogConfig) Adjust() { |
||||
if cfg.MaxFileSize <= 0 { |
||||
cfg.MaxFileSize = DefaultBinLogFileSize |
||||
} else if cfg.MaxFileSize > MaxBinLogFileSize { |
||||
cfg.MaxFileSize = MaxBinLogFileSize |
||||
} |
||||
|
||||
if cfg.MaxFileNum <= 0 { |
||||
cfg.MaxFileNum = DefaultBinLogFileNum |
||||
} else if cfg.MaxFileNum > MaxBinLogFileNum { |
||||
cfg.MaxFileNum = MaxBinLogFileNum |
||||
} |
||||
} |
||||
@ -0,0 +1,98 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"errors" |
||||
) |
||||
|
||||
const ( |
||||
NoneType byte = 0 |
||||
KVType byte = 1 |
||||
HashType byte = 2 |
||||
HSizeType byte = 3 |
||||
ListType byte = 4 |
||||
LMetaType byte = 5 |
||||
ZSetType byte = 6 |
||||
ZSizeType byte = 7 |
||||
ZScoreType byte = 8 |
||||
BitType byte = 9 |
||||
BitMetaType byte = 10 |
||||
SetType byte = 11 |
||||
SSizeType byte = 12 |
||||
|
||||
maxDataType byte = 100 |
||||
|
||||
ExpTimeType byte = 101 |
||||
ExpMetaType byte = 102 |
||||
) |
||||
|
||||
var ( |
||||
TypeName = map[byte]string{ |
||||
KVType: "kv", |
||||
HashType: "hash", |
||||
HSizeType: "hsize", |
||||
ListType: "list", |
||||
LMetaType: "lmeta", |
||||
ZSetType: "zset", |
||||
ZSizeType: "zsize", |
||||
ZScoreType: "zscore", |
||||
BitType: "bit", |
||||
BitMetaType: "bitmeta", |
||||
SetType: "set", |
||||
SSizeType: "ssize", |
||||
ExpTimeType: "exptime", |
||||
ExpMetaType: "expmeta", |
||||
} |
||||
) |
||||
|
||||
const ( |
||||
defaultScanCount int = 10 |
||||
) |
||||
|
||||
var ( |
||||
errKeySize = errors.New("invalid key size") |
||||
errValueSize = errors.New("invalid value size") |
||||
errHashFieldSize = errors.New("invalid hash field size") |
||||
errSetMemberSize = errors.New("invalid set member size") |
||||
errZSetMemberSize = errors.New("invalid zset member size") |
||||
errExpireValue = errors.New("invalid expire value") |
||||
) |
||||
|
||||
const ( |
||||
//we don't support too many databases
|
||||
MaxDBNumber uint8 = 16 |
||||
|
||||
//max key size
|
||||
MaxKeySize int = 1024 |
||||
|
||||
//max hash field size
|
||||
MaxHashFieldSize int = 1024 |
||||
|
||||
//max zset member size
|
||||
MaxZSetMemberSize int = 1024 |
||||
|
||||
//max set member size
|
||||
MaxSetMemberSize int = 1024 |
||||
|
||||
//max value size
|
||||
MaxValueSize int = 10 * 1024 * 1024 |
||||
) |
||||
|
||||
var ( |
||||
ErrScoreMiss = errors.New("zset score miss") |
||||
) |
||||
|
||||
const ( |
||||
BinLogTypeDeletion uint8 = 0x0 |
||||
BinLogTypePut uint8 = 0x1 |
||||
BinLogTypeCommand uint8 = 0x2 |
||||
) |
||||
|
||||
const ( |
||||
DBAutoCommit uint8 = 0x0 |
||||
DBInTransaction uint8 = 0x1 |
||||
DBInMulti uint8 = 0x2 |
||||
) |
||||
|
||||
var ( |
||||
Version = "0.1" |
||||
) |
||||
@ -0,0 +1,61 @@ |
||||
// package nodb is a high performance embedded NoSQL.
|
||||
//
|
||||
// nodb supports various data structure like kv, list, hash and zset like redis.
|
||||
//
|
||||
// Other features include binlog replication, data with a limited time-to-live.
|
||||
//
|
||||
// Usage
|
||||
//
|
||||
// First create a nodb instance before use:
|
||||
//
|
||||
// l := nodb.Open(cfg)
|
||||
//
|
||||
// cfg is a Config instance which contains configuration for nodb use,
|
||||
// like DataDir (root directory for nodb working to store data).
|
||||
//
|
||||
// After you create a nodb instance, you can select a DB to store you data:
|
||||
//
|
||||
// db, _ := l.Select(0)
|
||||
//
|
||||
// DB must be selected by a index, nodb supports only 16 databases, so the index range is [0-15].
|
||||
//
|
||||
// KV
|
||||
//
|
||||
// KV is the most basic nodb type like any other key-value database.
|
||||
//
|
||||
// err := db.Set(key, value)
|
||||
// value, err := db.Get(key)
|
||||
//
|
||||
// List
|
||||
//
|
||||
// List is simply lists of values, sorted by insertion order.
|
||||
// You can push or pop value on the list head (left) or tail (right).
|
||||
//
|
||||
// err := db.LPush(key, value1)
|
||||
// err := db.RPush(key, value2)
|
||||
// value1, err := db.LPop(key)
|
||||
// value2, err := db.RPop(key)
|
||||
//
|
||||
// Hash
|
||||
//
|
||||
// Hash is a map between fields and values.
|
||||
//
|
||||
// n, err := db.HSet(key, field1, value1)
|
||||
// n, err := db.HSet(key, field2, value2)
|
||||
// value1, err := db.HGet(key, field1)
|
||||
// value2, err := db.HGet(key, field2)
|
||||
//
|
||||
// ZSet
|
||||
//
|
||||
// ZSet is a sorted collections of values.
|
||||
// Every member of zset is associated with score, a int64 value which used to sort, from smallest to greatest score.
|
||||
// Members are unique, but score may be same.
|
||||
//
|
||||
// n, err := db.ZAdd(key, ScorePair{score1, member1}, ScorePair{score2, member2})
|
||||
// ay, err := db.ZRangeByScore(key, minScore, maxScore, 0, -1)
|
||||
//
|
||||
// Binlog
|
||||
//
|
||||
// nodb supports binlog, so you can sync binlog to another server for replication. If you want to open binlog support, set UseBinLog to true in config.
|
||||
//
|
||||
package nodb |
||||
@ -0,0 +1,200 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"bufio" |
||||
"bytes" |
||||
"encoding/binary" |
||||
"io" |
||||
"os" |
||||
|
||||
"github.com/siddontang/go-snappy/snappy" |
||||
) |
||||
|
||||
//dump format
|
||||
// fileIndex(bigendian int64)|filePos(bigendian int64)
|
||||
// |keylen(bigendian int32)|key|valuelen(bigendian int32)|value......
|
||||
//
|
||||
//key and value are both compressed for fast transfer dump on network using snappy
|
||||
|
||||
type BinLogAnchor struct { |
||||
LogFileIndex int64 |
||||
LogPos int64 |
||||
} |
||||
|
||||
func (m *BinLogAnchor) WriteTo(w io.Writer) error { |
||||
if err := binary.Write(w, binary.BigEndian, m.LogFileIndex); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if err := binary.Write(w, binary.BigEndian, m.LogPos); err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (m *BinLogAnchor) ReadFrom(r io.Reader) error { |
||||
err := binary.Read(r, binary.BigEndian, &m.LogFileIndex) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
err = binary.Read(r, binary.BigEndian, &m.LogPos) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (l *Nodb) DumpFile(path string) error { |
||||
f, err := os.Create(path) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer f.Close() |
||||
|
||||
return l.Dump(f) |
||||
} |
||||
|
||||
func (l *Nodb) Dump(w io.Writer) error { |
||||
m := new(BinLogAnchor) |
||||
|
||||
var err error |
||||
|
||||
l.wLock.Lock() |
||||
defer l.wLock.Unlock() |
||||
|
||||
if l.binlog != nil { |
||||
m.LogFileIndex = l.binlog.LogFileIndex() |
||||
m.LogPos = l.binlog.LogFilePos() |
||||
} |
||||
|
||||
wb := bufio.NewWriterSize(w, 4096) |
||||
if err = m.WriteTo(wb); err != nil { |
||||
return err |
||||
} |
||||
|
||||
it := l.ldb.NewIterator() |
||||
it.SeekToFirst() |
||||
|
||||
compressBuf := make([]byte, 4096) |
||||
|
||||
var key []byte |
||||
var value []byte |
||||
for ; it.Valid(); it.Next() { |
||||
key = it.RawKey() |
||||
value = it.RawValue() |
||||
|
||||
if key, err = snappy.Encode(compressBuf, key); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if err = binary.Write(wb, binary.BigEndian, uint16(len(key))); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if _, err = wb.Write(key); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if value, err = snappy.Encode(compressBuf, value); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if err = binary.Write(wb, binary.BigEndian, uint32(len(value))); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if _, err = wb.Write(value); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
if err = wb.Flush(); err != nil { |
||||
return err |
||||
} |
||||
|
||||
compressBuf = nil |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (l *Nodb) LoadDumpFile(path string) (*BinLogAnchor, error) { |
||||
f, err := os.Open(path) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
defer f.Close() |
||||
|
||||
return l.LoadDump(f) |
||||
} |
||||
|
||||
func (l *Nodb) LoadDump(r io.Reader) (*BinLogAnchor, error) { |
||||
l.wLock.Lock() |
||||
defer l.wLock.Unlock() |
||||
|
||||
info := new(BinLogAnchor) |
||||
|
||||
rb := bufio.NewReaderSize(r, 4096) |
||||
|
||||
err := info.ReadFrom(rb) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
var keyLen uint16 |
||||
var valueLen uint32 |
||||
|
||||
var keyBuf bytes.Buffer |
||||
var valueBuf bytes.Buffer |
||||
|
||||
deKeyBuf := make([]byte, 4096) |
||||
deValueBuf := make([]byte, 4096) |
||||
|
||||
var key, value []byte |
||||
|
||||
for { |
||||
if err = binary.Read(rb, binary.BigEndian, &keyLen); err != nil && err != io.EOF { |
||||
return nil, err |
||||
} else if err == io.EOF { |
||||
break |
||||
} |
||||
|
||||
if _, err = io.CopyN(&keyBuf, rb, int64(keyLen)); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if key, err = snappy.Decode(deKeyBuf, keyBuf.Bytes()); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if err = binary.Read(rb, binary.BigEndian, &valueLen); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if _, err = io.CopyN(&valueBuf, rb, int64(valueLen)); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if value, err = snappy.Decode(deValueBuf, valueBuf.Bytes()); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if err = l.ldb.Put(key, value); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
keyBuf.Reset() |
||||
valueBuf.Reset() |
||||
} |
||||
|
||||
deKeyBuf = nil |
||||
deValueBuf = nil |
||||
|
||||
//if binlog enable, we will delete all binlogs and open a new one for handling simply
|
||||
if l.binlog != nil { |
||||
l.binlog.PurgeAll() |
||||
} |
||||
|
||||
return info, nil |
||||
} |
||||
@ -0,0 +1,24 @@ |
||||
package nodb |
||||
|
||||
// todo, add info
|
||||
|
||||
// type Keyspace struct {
|
||||
// Kvs int `json:"kvs"`
|
||||
// KvExpires int `json:"kv_expires"`
|
||||
|
||||
// Lists int `json:"lists"`
|
||||
// ListExpires int `json:"list_expires"`
|
||||
|
||||
// Bitmaps int `json:"bitmaps"`
|
||||
// BitmapExpires int `json:"bitmap_expires"`
|
||||
|
||||
// ZSets int `json:"zsets"`
|
||||
// ZSetExpires int `json:"zset_expires"`
|
||||
|
||||
// Hashes int `json:"hashes"`
|
||||
// HashExpires int `json:"hahsh_expires"`
|
||||
// }
|
||||
|
||||
// type Info struct {
|
||||
// KeySpaces [MaxDBNumber]Keyspace
|
||||
// }
|
||||
@ -0,0 +1,73 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"errors" |
||||
"fmt" |
||||
) |
||||
|
||||
var ( |
||||
ErrNestMulti = errors.New("nest multi not supported") |
||||
ErrMultiDone = errors.New("multi has been closed") |
||||
) |
||||
|
||||
type Multi struct { |
||||
*DB |
||||
} |
||||
|
||||
func (db *DB) IsInMulti() bool { |
||||
return db.status == DBInMulti |
||||
} |
||||
|
||||
// begin a mutli to execute commands,
|
||||
// it will block any other write operations before you close the multi, unlike transaction, mutli can not rollback
|
||||
func (db *DB) Multi() (*Multi, error) { |
||||
if db.IsInMulti() { |
||||
return nil, ErrNestMulti |
||||
} |
||||
|
||||
m := new(Multi) |
||||
|
||||
m.DB = new(DB) |
||||
m.DB.status = DBInMulti |
||||
|
||||
m.DB.l = db.l |
||||
|
||||
m.l.wLock.Lock() |
||||
|
||||
m.DB.sdb = db.sdb |
||||
|
||||
m.DB.bucket = db.sdb |
||||
|
||||
m.DB.index = db.index |
||||
|
||||
m.DB.kvBatch = m.newBatch() |
||||
m.DB.listBatch = m.newBatch() |
||||
m.DB.hashBatch = m.newBatch() |
||||
m.DB.zsetBatch = m.newBatch() |
||||
m.DB.binBatch = m.newBatch() |
||||
m.DB.setBatch = m.newBatch() |
||||
|
||||
return m, nil |
||||
} |
||||
|
||||
func (m *Multi) newBatch() *batch { |
||||
return m.l.newBatch(m.bucket.NewWriteBatch(), &multiBatchLocker{}, nil) |
||||
} |
||||
|
||||
func (m *Multi) Close() error { |
||||
if m.bucket == nil { |
||||
return ErrMultiDone |
||||
} |
||||
m.l.wLock.Unlock() |
||||
m.bucket = nil |
||||
return nil |
||||
} |
||||
|
||||
func (m *Multi) Select(index int) error { |
||||
if index < 0 || index >= int(MaxDBNumber) { |
||||
return fmt.Errorf("invalid db index %d", index) |
||||
} |
||||
|
||||
m.DB.index = uint8(index) |
||||
return nil |
||||
} |
||||
@ -0,0 +1,128 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"fmt" |
||||
"sync" |
||||
"time" |
||||
|
||||
"github.com/lunny/log" |
||||
"github.com/lunny/nodb/config" |
||||
"github.com/lunny/nodb/store" |
||||
) |
||||
|
||||
type Nodb struct { |
||||
cfg *config.Config |
||||
|
||||
ldb *store.DB |
||||
dbs [MaxDBNumber]*DB |
||||
|
||||
quit chan struct{} |
||||
jobs *sync.WaitGroup |
||||
|
||||
binlog *BinLog |
||||
|
||||
wLock sync.RWMutex //allow one write at same time
|
||||
commitLock sync.Mutex //allow one write commit at same time
|
||||
} |
||||
|
||||
func Open(cfg *config.Config) (*Nodb, error) { |
||||
if len(cfg.DataDir) == 0 { |
||||
cfg.DataDir = config.DefaultDataDir |
||||
} |
||||
|
||||
ldb, err := store.Open(cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
l := new(Nodb) |
||||
|
||||
l.quit = make(chan struct{}) |
||||
l.jobs = new(sync.WaitGroup) |
||||
|
||||
l.ldb = ldb |
||||
|
||||
if cfg.BinLog.MaxFileNum > 0 && cfg.BinLog.MaxFileSize > 0 { |
||||
l.binlog, err = NewBinLog(cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} else { |
||||
l.binlog = nil |
||||
} |
||||
|
||||
for i := uint8(0); i < MaxDBNumber; i++ { |
||||
l.dbs[i] = l.newDB(i) |
||||
} |
||||
|
||||
l.activeExpireCycle() |
||||
|
||||
return l, nil |
||||
} |
||||
|
||||
func (l *Nodb) Close() { |
||||
close(l.quit) |
||||
l.jobs.Wait() |
||||
|
||||
l.ldb.Close() |
||||
|
||||
if l.binlog != nil { |
||||
l.binlog.Close() |
||||
l.binlog = nil |
||||
} |
||||
} |
||||
|
||||
func (l *Nodb) Select(index int) (*DB, error) { |
||||
if index < 0 || index >= int(MaxDBNumber) { |
||||
return nil, fmt.Errorf("invalid db index %d", index) |
||||
} |
||||
|
||||
return l.dbs[index], nil |
||||
} |
||||
|
||||
func (l *Nodb) FlushAll() error { |
||||
for index, db := range l.dbs { |
||||
if _, err := db.FlushAll(); err != nil { |
||||
log.Error("flush db %d error %s", index, err.Error()) |
||||
} |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// very dangerous to use
|
||||
func (l *Nodb) DataDB() *store.DB { |
||||
return l.ldb |
||||
} |
||||
|
||||
func (l *Nodb) activeExpireCycle() { |
||||
var executors []*elimination = make([]*elimination, len(l.dbs)) |
||||
for i, db := range l.dbs { |
||||
executors[i] = db.newEliminator() |
||||
} |
||||
|
||||
l.jobs.Add(1) |
||||
go func() { |
||||
tick := time.NewTicker(1 * time.Second) |
||||
end := false |
||||
done := make(chan struct{}) |
||||
for !end { |
||||
select { |
||||
case <-tick.C: |
||||
go func() { |
||||
for _, eli := range executors { |
||||
eli.active() |
||||
} |
||||
done <- struct{}{} |
||||
}() |
||||
<-done |
||||
case <-l.quit: |
||||
end = true |
||||
break |
||||
} |
||||
} |
||||
|
||||
tick.Stop() |
||||
l.jobs.Done() |
||||
}() |
||||
} |
||||
@ -0,0 +1,171 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"fmt" |
||||
"sync" |
||||
|
||||
"github.com/lunny/nodb/store" |
||||
) |
||||
|
||||
type ibucket interface { |
||||
Get(key []byte) ([]byte, error) |
||||
|
||||
Put(key []byte, value []byte) error |
||||
Delete(key []byte) error |
||||
|
||||
NewIterator() *store.Iterator |
||||
|
||||
NewWriteBatch() store.WriteBatch |
||||
|
||||
RangeIterator(min []byte, max []byte, rangeType uint8) *store.RangeLimitIterator |
||||
RevRangeIterator(min []byte, max []byte, rangeType uint8) *store.RangeLimitIterator |
||||
RangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *store.RangeLimitIterator |
||||
RevRangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *store.RangeLimitIterator |
||||
} |
||||
|
||||
type DB struct { |
||||
l *Nodb |
||||
|
||||
sdb *store.DB |
||||
|
||||
bucket ibucket |
||||
|
||||
index uint8 |
||||
|
||||
kvBatch *batch |
||||
listBatch *batch |
||||
hashBatch *batch |
||||
zsetBatch *batch |
||||
binBatch *batch |
||||
setBatch *batch |
||||
|
||||
status uint8 |
||||
} |
||||
|
||||
func (l *Nodb) newDB(index uint8) *DB { |
||||
d := new(DB) |
||||
|
||||
d.l = l |
||||
|
||||
d.sdb = l.ldb |
||||
|
||||
d.bucket = d.sdb |
||||
|
||||
d.status = DBAutoCommit |
||||
d.index = index |
||||
|
||||
d.kvBatch = d.newBatch() |
||||
d.listBatch = d.newBatch() |
||||
d.hashBatch = d.newBatch() |
||||
d.zsetBatch = d.newBatch() |
||||
d.binBatch = d.newBatch() |
||||
d.setBatch = d.newBatch() |
||||
|
||||
return d |
||||
} |
||||
|
||||
func (db *DB) newBatch() *batch { |
||||
return db.l.newBatch(db.bucket.NewWriteBatch(), &dbBatchLocker{l: &sync.Mutex{}, wrLock: &db.l.wLock}, nil) |
||||
} |
||||
|
||||
func (db *DB) Index() int { |
||||
return int(db.index) |
||||
} |
||||
|
||||
func (db *DB) IsAutoCommit() bool { |
||||
return db.status == DBAutoCommit |
||||
} |
||||
|
||||
func (db *DB) FlushAll() (drop int64, err error) { |
||||
all := [...](func() (int64, error)){ |
||||
db.flush, |
||||
db.lFlush, |
||||
db.hFlush, |
||||
db.zFlush, |
||||
db.bFlush, |
||||
db.sFlush} |
||||
|
||||
for _, flush := range all { |
||||
if n, e := flush(); e != nil { |
||||
err = e |
||||
return |
||||
} else { |
||||
drop += n |
||||
} |
||||
} |
||||
|
||||
return |
||||
} |
||||
|
||||
func (db *DB) newEliminator() *elimination { |
||||
eliminator := newEliminator(db) |
||||
|
||||
eliminator.regRetireContext(KVType, db.kvBatch, db.delete) |
||||
eliminator.regRetireContext(ListType, db.listBatch, db.lDelete) |
||||
eliminator.regRetireContext(HashType, db.hashBatch, db.hDelete) |
||||
eliminator.regRetireContext(ZSetType, db.zsetBatch, db.zDelete) |
||||
eliminator.regRetireContext(BitType, db.binBatch, db.bDelete) |
||||
eliminator.regRetireContext(SetType, db.setBatch, db.sDelete) |
||||
|
||||
return eliminator |
||||
} |
||||
|
||||
func (db *DB) flushRegion(t *batch, minKey []byte, maxKey []byte) (drop int64, err error) { |
||||
it := db.bucket.RangeIterator(minKey, maxKey, store.RangeROpen) |
||||
for ; it.Valid(); it.Next() { |
||||
t.Delete(it.RawKey()) |
||||
drop++ |
||||
if drop&1023 == 0 { |
||||
if err = t.Commit(); err != nil { |
||||
return |
||||
} |
||||
} |
||||
} |
||||
it.Close() |
||||
return |
||||
} |
||||
|
||||
func (db *DB) flushType(t *batch, dataType byte) (drop int64, err error) { |
||||
var deleteFunc func(t *batch, key []byte) int64 |
||||
var metaDataType byte |
||||
switch dataType { |
||||
case KVType: |
||||
deleteFunc = db.delete |
||||
metaDataType = KVType |
||||
case ListType: |
||||
deleteFunc = db.lDelete |
||||
metaDataType = LMetaType |
||||
case HashType: |
||||
deleteFunc = db.hDelete |
||||
metaDataType = HSizeType |
||||
case ZSetType: |
||||
deleteFunc = db.zDelete |
||||
metaDataType = ZSizeType |
||||
case BitType: |
||||
deleteFunc = db.bDelete |
||||
metaDataType = BitMetaType |
||||
case SetType: |
||||
deleteFunc = db.sDelete |
||||
metaDataType = SSizeType |
||||
default: |
||||
return 0, fmt.Errorf("invalid data type: %s", TypeName[dataType]) |
||||
} |
||||
|
||||
var keys [][]byte |
||||
keys, err = db.scan(metaDataType, nil, 1024, false, "") |
||||
for len(keys) != 0 || err != nil { |
||||
for _, key := range keys { |
||||
deleteFunc(t, key) |
||||
db.rmExpire(t, dataType, key) |
||||
|
||||
} |
||||
|
||||
if err = t.Commit(); err != nil { |
||||
return |
||||
} else { |
||||
drop += int64(len(keys)) |
||||
} |
||||
keys, err = db.scan(metaDataType, nil, 1024, false, "") |
||||
} |
||||
return |
||||
} |
||||
@ -0,0 +1,312 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"bufio" |
||||
"bytes" |
||||
"errors" |
||||
"io" |
||||
"os" |
||||
"time" |
||||
|
||||
"github.com/lunny/log" |
||||
"github.com/lunny/nodb/store/driver" |
||||
) |
||||
|
||||
const ( |
||||
maxReplBatchNum = 100 |
||||
maxReplLogSize = 1 * 1024 * 1024 |
||||
) |
||||
|
||||
var ( |
||||
ErrSkipEvent = errors.New("skip to next event") |
||||
) |
||||
|
||||
var ( |
||||
errInvalidBinLogEvent = errors.New("invalid binglog event") |
||||
errInvalidBinLogFile = errors.New("invalid binlog file") |
||||
) |
||||
|
||||
type replBatch struct { |
||||
wb driver.IWriteBatch |
||||
events [][]byte |
||||
l *Nodb |
||||
|
||||
lastHead *BinLogHead |
||||
} |
||||
|
||||
func (b *replBatch) Commit() error { |
||||
b.l.commitLock.Lock() |
||||
defer b.l.commitLock.Unlock() |
||||
|
||||
err := b.wb.Commit() |
||||
if err != nil { |
||||
b.Rollback() |
||||
return err |
||||
} |
||||
|
||||
if b.l.binlog != nil { |
||||
if err = b.l.binlog.Log(b.events...); err != nil { |
||||
b.Rollback() |
||||
return err |
||||
} |
||||
} |
||||
|
||||
b.events = [][]byte{} |
||||
b.lastHead = nil |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (b *replBatch) Rollback() error { |
||||
b.wb.Rollback() |
||||
b.events = [][]byte{} |
||||
b.lastHead = nil |
||||
return nil |
||||
} |
||||
|
||||
func (l *Nodb) replicateEvent(b *replBatch, event []byte) error { |
||||
if len(event) == 0 { |
||||
return errInvalidBinLogEvent |
||||
} |
||||
|
||||
b.events = append(b.events, event) |
||||
|
||||
logType := uint8(event[0]) |
||||
switch logType { |
||||
case BinLogTypePut: |
||||
return l.replicatePutEvent(b, event) |
||||
case BinLogTypeDeletion: |
||||
return l.replicateDeleteEvent(b, event) |
||||
default: |
||||
return errInvalidBinLogEvent |
||||
} |
||||
} |
||||
|
||||
func (l *Nodb) replicatePutEvent(b *replBatch, event []byte) error { |
||||
key, value, err := decodeBinLogPut(event) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
b.wb.Put(key, value) |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (l *Nodb) replicateDeleteEvent(b *replBatch, event []byte) error { |
||||
key, err := decodeBinLogDelete(event) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
b.wb.Delete(key) |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func ReadEventFromReader(rb io.Reader, f func(head *BinLogHead, event []byte) error) error { |
||||
head := &BinLogHead{} |
||||
var err error |
||||
|
||||
for { |
||||
if err = head.Read(rb); err != nil { |
||||
if err == io.EOF { |
||||
break |
||||
} else { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
var dataBuf bytes.Buffer |
||||
|
||||
if _, err = io.CopyN(&dataBuf, rb, int64(head.PayloadLen)); err != nil { |
||||
return err |
||||
} |
||||
|
||||
err = f(head, dataBuf.Bytes()) |
||||
if err != nil && err != ErrSkipEvent { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (l *Nodb) ReplicateFromReader(rb io.Reader) error { |
||||
b := new(replBatch) |
||||
|
||||
b.wb = l.ldb.NewWriteBatch() |
||||
b.l = l |
||||
|
||||
f := func(head *BinLogHead, event []byte) error { |
||||
if b.lastHead == nil { |
||||
b.lastHead = head |
||||
} else if !b.lastHead.InSameBatch(head) { |
||||
if err := b.Commit(); err != nil { |
||||
log.Fatal("replication error %s, skip to next", err.Error()) |
||||
return ErrSkipEvent |
||||
} |
||||
b.lastHead = head |
||||
} |
||||
|
||||
err := l.replicateEvent(b, event) |
||||
if err != nil { |
||||
log.Fatal("replication error %s, skip to next", err.Error()) |
||||
return ErrSkipEvent |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
err := ReadEventFromReader(rb, f) |
||||
if err != nil { |
||||
b.Rollback() |
||||
return err |
||||
} |
||||
return b.Commit() |
||||
} |
||||
|
||||
func (l *Nodb) ReplicateFromData(data []byte) error { |
||||
rb := bytes.NewReader(data) |
||||
|
||||
err := l.ReplicateFromReader(rb) |
||||
|
||||
return err |
||||
} |
||||
|
||||
func (l *Nodb) ReplicateFromBinLog(filePath string) error { |
||||
f, err := os.Open(filePath) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
rb := bufio.NewReaderSize(f, 4096) |
||||
|
||||
err = l.ReplicateFromReader(rb) |
||||
|
||||
f.Close() |
||||
|
||||
return err |
||||
} |
||||
|
||||
// try to read events, if no events read, try to wait the new event singal until timeout seconds
|
||||
func (l *Nodb) ReadEventsToTimeout(info *BinLogAnchor, w io.Writer, timeout int) (n int, err error) { |
||||
lastIndex := info.LogFileIndex |
||||
lastPos := info.LogPos |
||||
|
||||
n = 0 |
||||
if l.binlog == nil { |
||||
//binlog not supported
|
||||
info.LogFileIndex = 0 |
||||
info.LogPos = 0 |
||||
return |
||||
} |
||||
|
||||
n, err = l.ReadEventsTo(info, w) |
||||
if err == nil && info.LogFileIndex == lastIndex && info.LogPos == lastPos { |
||||
//no events read
|
||||
select { |
||||
case <-l.binlog.Wait(): |
||||
case <-time.After(time.Duration(timeout) * time.Second): |
||||
} |
||||
return l.ReadEventsTo(info, w) |
||||
} |
||||
return |
||||
} |
||||
|
||||
func (l *Nodb) ReadEventsTo(info *BinLogAnchor, w io.Writer) (n int, err error) { |
||||
n = 0 |
||||
if l.binlog == nil { |
||||
//binlog not supported
|
||||
info.LogFileIndex = 0 |
||||
info.LogPos = 0 |
||||
return |
||||
} |
||||
|
||||
index := info.LogFileIndex |
||||
offset := info.LogPos |
||||
|
||||
filePath := l.binlog.FormatLogFilePath(index) |
||||
|
||||
var f *os.File |
||||
f, err = os.Open(filePath) |
||||
if os.IsNotExist(err) { |
||||
lastIndex := l.binlog.LogFileIndex() |
||||
|
||||
if index == lastIndex { |
||||
//no binlog at all
|
||||
info.LogPos = 0 |
||||
} else { |
||||
//slave binlog info had lost
|
||||
info.LogFileIndex = -1 |
||||
} |
||||
} |
||||
|
||||
if err != nil { |
||||
if os.IsNotExist(err) { |
||||
err = nil |
||||
} |
||||
return |
||||
} |
||||
|
||||
defer f.Close() |
||||
|
||||
var fileSize int64 |
||||
st, _ := f.Stat() |
||||
fileSize = st.Size() |
||||
|
||||
if fileSize == info.LogPos { |
||||
return |
||||
} |
||||
|
||||
if _, err = f.Seek(offset, os.SEEK_SET); err != nil { |
||||
//may be invliad seek offset
|
||||
return |
||||
} |
||||
|
||||
var lastHead *BinLogHead = nil |
||||
|
||||
head := &BinLogHead{} |
||||
|
||||
batchNum := 0 |
||||
|
||||
for { |
||||
if err = head.Read(f); err != nil { |
||||
if err == io.EOF { |
||||
//we will try to use next binlog
|
||||
if index < l.binlog.LogFileIndex() { |
||||
info.LogFileIndex += 1 |
||||
info.LogPos = 0 |
||||
} |
||||
err = nil |
||||
return |
||||
} else { |
||||
return |
||||
} |
||||
|
||||
} |
||||
|
||||
if lastHead == nil { |
||||
lastHead = head |
||||
batchNum++ |
||||
} else if !lastHead.InSameBatch(head) { |
||||
lastHead = head |
||||
batchNum++ |
||||
if batchNum > maxReplBatchNum || n > maxReplLogSize { |
||||
return |
||||
} |
||||
} |
||||
|
||||
if err = head.Write(w); err != nil { |
||||
return |
||||
} |
||||
|
||||
if _, err = io.CopyN(w, f, int64(head.PayloadLen)); err != nil { |
||||
return |
||||
} |
||||
|
||||
n += (head.Len() + int(head.PayloadLen)) |
||||
info.LogPos = info.LogPos + int64(head.Len()) + int64(head.PayloadLen) |
||||
} |
||||
|
||||
return |
||||
} |
||||
@ -0,0 +1,144 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"bytes" |
||||
"errors" |
||||
"regexp" |
||||
|
||||
"github.com/lunny/nodb/store" |
||||
) |
||||
|
||||
var errDataType = errors.New("error data type") |
||||
var errMetaKey = errors.New("error meta key") |
||||
|
||||
// Seek search the prefix key
|
||||
func (db *DB) Seek(key []byte) (*store.Iterator, error) { |
||||
return db.seek(KVType, key) |
||||
} |
||||
|
||||
func (db *DB) seek(dataType byte, key []byte) (*store.Iterator, error) { |
||||
var minKey []byte |
||||
var err error |
||||
|
||||
if len(key) > 0 { |
||||
if err = checkKeySize(key); err != nil { |
||||
return nil, err |
||||
} |
||||
if minKey, err = db.encodeMetaKey(dataType, key); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
} else { |
||||
if minKey, err = db.encodeMinKey(dataType); err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
it := db.bucket.NewIterator() |
||||
it.Seek(minKey) |
||||
return it, nil |
||||
} |
||||
|
||||
func (db *DB) MaxKey() ([]byte, error) { |
||||
return db.encodeMaxKey(KVType) |
||||
} |
||||
|
||||
func (db *DB) Key(it *store.Iterator) ([]byte, error) { |
||||
return db.decodeMetaKey(KVType, it.Key()) |
||||
} |
||||
|
||||
func (db *DB) scan(dataType byte, key []byte, count int, inclusive bool, match string) ([][]byte, error) { |
||||
var minKey, maxKey []byte |
||||
var err error |
||||
var r *regexp.Regexp |
||||
|
||||
if len(match) > 0 { |
||||
if r, err = regexp.Compile(match); err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
if len(key) > 0 { |
||||
if err = checkKeySize(key); err != nil { |
||||
return nil, err |
||||
} |
||||
if minKey, err = db.encodeMetaKey(dataType, key); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
} else { |
||||
if minKey, err = db.encodeMinKey(dataType); err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
if maxKey, err = db.encodeMaxKey(dataType); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if count <= 0 { |
||||
count = defaultScanCount |
||||
} |
||||
|
||||
v := make([][]byte, 0, count) |
||||
|
||||
it := db.bucket.NewIterator() |
||||
it.Seek(minKey) |
||||
|
||||
if !inclusive { |
||||
if it.Valid() && bytes.Equal(it.RawKey(), minKey) { |
||||
it.Next() |
||||
} |
||||
} |
||||
|
||||
for i := 0; it.Valid() && i < count && bytes.Compare(it.RawKey(), maxKey) < 0; it.Next() { |
||||
if k, err := db.decodeMetaKey(dataType, it.Key()); err != nil { |
||||
continue |
||||
} else if r != nil && !r.Match(k) { |
||||
continue |
||||
} else { |
||||
v = append(v, k) |
||||
i++ |
||||
} |
||||
} |
||||
it.Close() |
||||
return v, nil |
||||
} |
||||
|
||||
func (db *DB) encodeMinKey(dataType byte) ([]byte, error) { |
||||
return db.encodeMetaKey(dataType, nil) |
||||
} |
||||
|
||||
func (db *DB) encodeMaxKey(dataType byte) ([]byte, error) { |
||||
k, err := db.encodeMetaKey(dataType, nil) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
k[len(k)-1] = dataType + 1 |
||||
return k, nil |
||||
} |
||||
|
||||
func (db *DB) encodeMetaKey(dataType byte, key []byte) ([]byte, error) { |
||||
switch dataType { |
||||
case KVType: |
||||
return db.encodeKVKey(key), nil |
||||
case LMetaType: |
||||
return db.lEncodeMetaKey(key), nil |
||||
case HSizeType: |
||||
return db.hEncodeSizeKey(key), nil |
||||
case ZSizeType: |
||||
return db.zEncodeSizeKey(key), nil |
||||
case BitMetaType: |
||||
return db.bEncodeMetaKey(key), nil |
||||
case SSizeType: |
||||
return db.sEncodeSizeKey(key), nil |
||||
default: |
||||
return nil, errDataType |
||||
} |
||||
} |
||||
func (db *DB) decodeMetaKey(dataType byte, ek []byte) ([]byte, error) { |
||||
if len(ek) < 2 || ek[0] != db.index || ek[1] != dataType { |
||||
return nil, errMetaKey |
||||
} |
||||
return ek[2:], nil |
||||
} |
||||
@ -0,0 +1,61 @@ |
||||
package store |
||||
|
||||
import ( |
||||
"github.com/lunny/nodb/store/driver" |
||||
) |
||||
|
||||
type DB struct { |
||||
driver.IDB |
||||
} |
||||
|
||||
func (db *DB) NewIterator() *Iterator { |
||||
it := new(Iterator) |
||||
it.it = db.IDB.NewIterator() |
||||
|
||||
return it |
||||
} |
||||
|
||||
func (db *DB) NewWriteBatch() WriteBatch { |
||||
return db.IDB.NewWriteBatch() |
||||
} |
||||
|
||||
func (db *DB) NewSnapshot() (*Snapshot, error) { |
||||
var err error |
||||
s := &Snapshot{} |
||||
if s.ISnapshot, err = db.IDB.NewSnapshot(); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return s, nil |
||||
} |
||||
|
||||
func (db *DB) RangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator { |
||||
return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{0, -1}) |
||||
} |
||||
|
||||
func (db *DB) RevRangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator { |
||||
return NewRevRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{0, -1}) |
||||
} |
||||
|
||||
//count < 0, unlimit.
|
||||
//
|
||||
//offset must >= 0, if < 0, will get nothing.
|
||||
func (db *DB) RangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *RangeLimitIterator { |
||||
return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{offset, count}) |
||||
} |
||||
|
||||
//count < 0, unlimit.
|
||||
//
|
||||
//offset must >= 0, if < 0, will get nothing.
|
||||
func (db *DB) RevRangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *RangeLimitIterator { |
||||
return NewRevRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{offset, count}) |
||||
} |
||||
|
||||
func (db *DB) Begin() (*Tx, error) { |
||||
tx, err := db.IDB.Begin() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return &Tx{tx}, nil |
||||
} |
||||
@ -0,0 +1,39 @@ |
||||
package driver |
||||
|
||||
type BatchPuter interface { |
||||
BatchPut([]Write) error |
||||
} |
||||
|
||||
type Write struct { |
||||
Key []byte |
||||
Value []byte |
||||
} |
||||
|
||||
type WriteBatch struct { |
||||
batch BatchPuter |
||||
wb []Write |
||||
} |
||||
|
||||
func (w *WriteBatch) Put(key, value []byte) { |
||||
if value == nil { |
||||
value = []byte{} |
||||
} |
||||
w.wb = append(w.wb, Write{key, value}) |
||||
} |
||||
|
||||
func (w *WriteBatch) Delete(key []byte) { |
||||
w.wb = append(w.wb, Write{key, nil}) |
||||
} |
||||
|
||||
func (w *WriteBatch) Commit() error { |
||||
return w.batch.BatchPut(w.wb) |
||||
} |
||||
|
||||
func (w *WriteBatch) Rollback() error { |
||||
w.wb = w.wb[0:0] |
||||
return nil |
||||
} |
||||
|
||||
func NewWriteBatch(puter BatchPuter) IWriteBatch { |
||||
return &WriteBatch{puter, []Write{}} |
||||
} |
||||
@ -0,0 +1,67 @@ |
||||
package driver |
||||
|
||||
import ( |
||||
"errors" |
||||
) |
||||
|
||||
var ( |
||||
ErrTxSupport = errors.New("transaction is not supported") |
||||
) |
||||
|
||||
type IDB interface { |
||||
Close() error |
||||
|
||||
Get(key []byte) ([]byte, error) |
||||
|
||||
Put(key []byte, value []byte) error |
||||
Delete(key []byte) error |
||||
|
||||
NewIterator() IIterator |
||||
|
||||
NewWriteBatch() IWriteBatch |
||||
|
||||
NewSnapshot() (ISnapshot, error) |
||||
|
||||
Begin() (Tx, error) |
||||
} |
||||
|
||||
type ISnapshot interface { |
||||
Get(key []byte) ([]byte, error) |
||||
NewIterator() IIterator |
||||
Close() |
||||
} |
||||
|
||||
type IIterator interface { |
||||
Close() error |
||||
|
||||
First() |
||||
Last() |
||||
Seek(key []byte) |
||||
|
||||
Next() |
||||
Prev() |
||||
|
||||
Valid() bool |
||||
|
||||
Key() []byte |
||||
Value() []byte |
||||
} |
||||
|
||||
type IWriteBatch interface { |
||||
Put(key []byte, value []byte) |
||||
Delete(key []byte) |
||||
Commit() error |
||||
Rollback() error |
||||
} |
||||
|
||||
type Tx interface { |
||||
Get(key []byte) ([]byte, error) |
||||
Put(key []byte, value []byte) error |
||||
Delete(key []byte) error |
||||
|
||||
NewIterator() IIterator |
||||
NewWriteBatch() IWriteBatch |
||||
|
||||
Commit() error |
||||
Rollback() error |
||||
} |
||||
@ -0,0 +1,46 @@ |
||||
package driver |
||||
|
||||
import ( |
||||
"fmt" |
||||
|
||||
"github.com/lunny/nodb/config" |
||||
) |
||||
|
||||
type Store interface { |
||||
String() string |
||||
Open(path string, cfg *config.Config) (IDB, error) |
||||
Repair(path string, cfg *config.Config) error |
||||
} |
||||
|
||||
var dbs = map[string]Store{} |
||||
|
||||
func Register(s Store) { |
||||
name := s.String() |
||||
if _, ok := dbs[name]; ok { |
||||
panic(fmt.Errorf("store %s is registered", s)) |
||||
} |
||||
|
||||
dbs[name] = s |
||||
} |
||||
|
||||
func ListStores() []string { |
||||
s := []string{} |
||||
for k, _ := range dbs { |
||||
s = append(s, k) |
||||
} |
||||
|
||||
return s |
||||
} |
||||
|
||||
func GetStore(cfg *config.Config) (Store, error) { |
||||
if len(cfg.DBName) == 0 { |
||||
cfg.DBName = config.DefaultDBName |
||||
} |
||||
|
||||
s, ok := dbs[cfg.DBName] |
||||
if !ok { |
||||
return nil, fmt.Errorf("store %s is not registered", cfg.DBName) |
||||
} |
||||
|
||||
return s, nil |
||||
} |
||||
@ -0,0 +1,27 @@ |
||||
package goleveldb |
||||
|
||||
import ( |
||||
"github.com/syndtr/goleveldb/leveldb" |
||||
) |
||||
|
||||
type WriteBatch struct { |
||||
db *DB |
||||
wbatch *leveldb.Batch |
||||
} |
||||
|
||||
func (w *WriteBatch) Put(key, value []byte) { |
||||
w.wbatch.Put(key, value) |
||||
} |
||||
|
||||
func (w *WriteBatch) Delete(key []byte) { |
||||
w.wbatch.Delete(key) |
||||
} |
||||
|
||||
func (w *WriteBatch) Commit() error { |
||||
return w.db.db.Write(w.wbatch, nil) |
||||
} |
||||
|
||||
func (w *WriteBatch) Rollback() error { |
||||
w.wbatch.Reset() |
||||
return nil |
||||
} |
||||
@ -0,0 +1,4 @@ |
||||
package goleveldb |
||||
|
||||
const DBName = "goleveldb" |
||||
const MemDBName = "memory" |
||||
@ -0,0 +1,187 @@ |
||||
package goleveldb |
||||
|
||||
import ( |
||||
"github.com/syndtr/goleveldb/leveldb" |
||||
"github.com/syndtr/goleveldb/leveldb/cache" |
||||
"github.com/syndtr/goleveldb/leveldb/filter" |
||||
"github.com/syndtr/goleveldb/leveldb/opt" |
||||
"github.com/syndtr/goleveldb/leveldb/storage" |
||||
|
||||
"github.com/lunny/nodb/config" |
||||
"github.com/lunny/nodb/store/driver" |
||||
|
||||
"os" |
||||
) |
||||
|
||||
const defaultFilterBits int = 10 |
||||
|
||||
type Store struct { |
||||
} |
||||
|
||||
func (s Store) String() string { |
||||
return DBName |
||||
} |
||||
|
||||
type MemStore struct { |
||||
} |
||||
|
||||
func (s MemStore) String() string { |
||||
return MemDBName |
||||
} |
||||
|
||||
type DB struct { |
||||
path string |
||||
|
||||
cfg *config.LevelDBConfig |
||||
|
||||
db *leveldb.DB |
||||
|
||||
opts *opt.Options |
||||
|
||||
iteratorOpts *opt.ReadOptions |
||||
|
||||
cache cache.Cache |
||||
|
||||
filter filter.Filter |
||||
} |
||||
|
||||
func (s Store) Open(path string, cfg *config.Config) (driver.IDB, error) { |
||||
if err := os.MkdirAll(path, os.ModePerm); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
db := new(DB) |
||||
db.path = path |
||||
db.cfg = &cfg.LevelDB |
||||
|
||||
db.initOpts() |
||||
|
||||
var err error |
||||
db.db, err = leveldb.OpenFile(db.path, db.opts) |
||||
|
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return db, nil |
||||
} |
||||
|
||||
func (s Store) Repair(path string, cfg *config.Config) error { |
||||
db, err := leveldb.RecoverFile(path, newOptions(&cfg.LevelDB)) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
db.Close() |
||||
return nil |
||||
} |
||||
|
||||
func (s MemStore) Open(path string, cfg *config.Config) (driver.IDB, error) { |
||||
db := new(DB) |
||||
db.path = path |
||||
db.cfg = &cfg.LevelDB |
||||
|
||||
db.initOpts() |
||||
|
||||
var err error |
||||
db.db, err = leveldb.Open(storage.NewMemStorage(), db.opts) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return db, nil |
||||
} |
||||
|
||||
func (s MemStore) Repair(path string, cfg *config.Config) error { |
||||
return nil |
||||
} |
||||
|
||||
func (db *DB) initOpts() { |
||||
db.opts = newOptions(db.cfg) |
||||
|
||||
db.iteratorOpts = &opt.ReadOptions{} |
||||
db.iteratorOpts.DontFillCache = true |
||||
} |
||||
|
||||
func newOptions(cfg *config.LevelDBConfig) *opt.Options { |
||||
opts := &opt.Options{} |
||||
opts.ErrorIfMissing = false |
||||
|
||||
cfg.Adjust() |
||||
|
||||
//opts.BlockCacher = cache.NewLRU(cfg.CacheSize)
|
||||
opts.BlockCacheCapacity = cfg.CacheSize |
||||
|
||||
//we must use bloomfilter
|
||||
opts.Filter = filter.NewBloomFilter(defaultFilterBits) |
||||
|
||||
if !cfg.Compression { |
||||
opts.Compression = opt.NoCompression |
||||
} else { |
||||
opts.Compression = opt.SnappyCompression |
||||
} |
||||
|
||||
opts.BlockSize = cfg.BlockSize |
||||
opts.WriteBuffer = cfg.WriteBufferSize |
||||
|
||||
return opts |
||||
} |
||||
|
||||
func (db *DB) Close() error { |
||||
return db.db.Close() |
||||
} |
||||
|
||||
func (db *DB) Put(key, value []byte) error { |
||||
return db.db.Put(key, value, nil) |
||||
} |
||||
|
||||
func (db *DB) Get(key []byte) ([]byte, error) { |
||||
v, err := db.db.Get(key, nil) |
||||
if err == leveldb.ErrNotFound { |
||||
return nil, nil |
||||
} |
||||
return v, nil |
||||
} |
||||
|
||||
func (db *DB) Delete(key []byte) error { |
||||
return db.db.Delete(key, nil) |
||||
} |
||||
|
||||
func (db *DB) NewWriteBatch() driver.IWriteBatch { |
||||
wb := &WriteBatch{ |
||||
db: db, |
||||
wbatch: new(leveldb.Batch), |
||||
} |
||||
return wb |
||||
} |
||||
|
||||
func (db *DB) NewIterator() driver.IIterator { |
||||
it := &Iterator{ |
||||
db.db.NewIterator(nil, db.iteratorOpts), |
||||
} |
||||
|
||||
return it |
||||
} |
||||
|
||||
func (db *DB) Begin() (driver.Tx, error) { |
||||
return nil, driver.ErrTxSupport |
||||
} |
||||
|
||||
func (db *DB) NewSnapshot() (driver.ISnapshot, error) { |
||||
snapshot, err := db.db.GetSnapshot() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
s := &Snapshot{ |
||||
db: db, |
||||
snp: snapshot, |
||||
} |
||||
|
||||
return s, nil |
||||
} |
||||
|
||||
func init() { |
||||
driver.Register(Store{}) |
||||
driver.Register(MemStore{}) |
||||
} |
||||
@ -0,0 +1,49 @@ |
||||
package goleveldb |
||||
|
||||
import ( |
||||
"github.com/syndtr/goleveldb/leveldb/iterator" |
||||
) |
||||
|
||||
type Iterator struct { |
||||
it iterator.Iterator |
||||
} |
||||
|
||||
func (it *Iterator) Key() []byte { |
||||
return it.it.Key() |
||||
} |
||||
|
||||
func (it *Iterator) Value() []byte { |
||||
return it.it.Value() |
||||
} |
||||
|
||||
func (it *Iterator) Close() error { |
||||
if it.it != nil { |
||||
it.it.Release() |
||||
it.it = nil |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (it *Iterator) Valid() bool { |
||||
return it.it.Valid() |
||||
} |
||||
|
||||
func (it *Iterator) Next() { |
||||
it.it.Next() |
||||
} |
||||
|
||||
func (it *Iterator) Prev() { |
||||
it.it.Prev() |
||||
} |
||||
|
||||
func (it *Iterator) First() { |
||||
it.it.First() |
||||
} |
||||
|
||||
func (it *Iterator) Last() { |
||||
it.it.Last() |
||||
} |
||||
|
||||
func (it *Iterator) Seek(key []byte) { |
||||
it.it.Seek(key) |
||||
} |
||||
@ -0,0 +1,26 @@ |
||||
package goleveldb |
||||
|
||||
import ( |
||||
"github.com/lunny/nodb/store/driver" |
||||
"github.com/syndtr/goleveldb/leveldb" |
||||
) |
||||
|
||||
type Snapshot struct { |
||||
db *DB |
||||
snp *leveldb.Snapshot |
||||
} |
||||
|
||||
func (s *Snapshot) Get(key []byte) ([]byte, error) { |
||||
return s.snp.Get(key, s.db.iteratorOpts) |
||||
} |
||||
|
||||
func (s *Snapshot) NewIterator() driver.IIterator { |
||||
it := &Iterator{ |
||||
s.snp.NewIterator(nil, s.db.iteratorOpts), |
||||
} |
||||
return it |
||||
} |
||||
|
||||
func (s *Snapshot) Close() { |
||||
s.snp.Release() |
||||
} |
||||
@ -0,0 +1,327 @@ |
||||
package store |
||||
|
||||
import ( |
||||
"bytes" |
||||
|
||||
"github.com/lunny/nodb/store/driver" |
||||
) |
||||
|
||||
const ( |
||||
IteratorForward uint8 = 0 |
||||
IteratorBackward uint8 = 1 |
||||
) |
||||
|
||||
const ( |
||||
RangeClose uint8 = 0x00 |
||||
RangeLOpen uint8 = 0x01 |
||||
RangeROpen uint8 = 0x10 |
||||
RangeOpen uint8 = 0x11 |
||||
) |
||||
|
||||
// min must less or equal than max
|
||||
//
|
||||
// range type:
|
||||
//
|
||||
// close: [min, max]
|
||||
// open: (min, max)
|
||||
// lopen: (min, max]
|
||||
// ropen: [min, max)
|
||||
//
|
||||
type Range struct { |
||||
Min []byte |
||||
Max []byte |
||||
|
||||
Type uint8 |
||||
} |
||||
|
||||
type Limit struct { |
||||
Offset int |
||||
Count int |
||||
} |
||||
|
||||
type Iterator struct { |
||||
it driver.IIterator |
||||
} |
||||
|
||||
// Returns a copy of key.
|
||||
func (it *Iterator) Key() []byte { |
||||
k := it.it.Key() |
||||
if k == nil { |
||||
return nil |
||||
} |
||||
|
||||
return append([]byte{}, k...) |
||||
} |
||||
|
||||
// Returns a copy of value.
|
||||
func (it *Iterator) Value() []byte { |
||||
v := it.it.Value() |
||||
if v == nil { |
||||
return nil |
||||
} |
||||
|
||||
return append([]byte{}, v...) |
||||
} |
||||
|
||||
// Returns a reference of key.
|
||||
// you must be careful that it will be changed after next iterate.
|
||||
func (it *Iterator) RawKey() []byte { |
||||
return it.it.Key() |
||||
} |
||||
|
||||
// Returns a reference of value.
|
||||
// you must be careful that it will be changed after next iterate.
|
||||
func (it *Iterator) RawValue() []byte { |
||||
return it.it.Value() |
||||
} |
||||
|
||||
// Copy key to b, if b len is small or nil, returns a new one.
|
||||
func (it *Iterator) BufKey(b []byte) []byte { |
||||
k := it.RawKey() |
||||
if k == nil { |
||||
return nil |
||||
} |
||||
if b == nil { |
||||
b = []byte{} |
||||
} |
||||
|
||||
b = b[0:0] |
||||
return append(b, k...) |
||||
} |
||||
|
||||
// Copy value to b, if b len is small or nil, returns a new one.
|
||||
func (it *Iterator) BufValue(b []byte) []byte { |
||||
v := it.RawValue() |
||||
if v == nil { |
||||
return nil |
||||
} |
||||
|
||||
if b == nil { |
||||
b = []byte{} |
||||
} |
||||
|
||||
b = b[0:0] |
||||
return append(b, v...) |
||||
} |
||||
|
||||
func (it *Iterator) Close() { |
||||
if it.it != nil { |
||||
it.it.Close() |
||||
it.it = nil |
||||
} |
||||
} |
||||
|
||||
func (it *Iterator) Valid() bool { |
||||
return it.it.Valid() |
||||
} |
||||
|
||||
func (it *Iterator) Next() { |
||||
it.it.Next() |
||||
} |
||||
|
||||
func (it *Iterator) Prev() { |
||||
it.it.Prev() |
||||
} |
||||
|
||||
func (it *Iterator) SeekToFirst() { |
||||
it.it.First() |
||||
} |
||||
|
||||
func (it *Iterator) SeekToLast() { |
||||
it.it.Last() |
||||
} |
||||
|
||||
func (it *Iterator) Seek(key []byte) { |
||||
it.it.Seek(key) |
||||
} |
||||
|
||||
// Finds by key, if not found, nil returns.
|
||||
func (it *Iterator) Find(key []byte) []byte { |
||||
it.Seek(key) |
||||
if it.Valid() { |
||||
k := it.RawKey() |
||||
if k == nil { |
||||
return nil |
||||
} else if bytes.Equal(k, key) { |
||||
return it.Value() |
||||
} |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// Finds by key, if not found, nil returns, else a reference of value returns.
|
||||
// you must be careful that it will be changed after next iterate.
|
||||
func (it *Iterator) RawFind(key []byte) []byte { |
||||
it.Seek(key) |
||||
if it.Valid() { |
||||
k := it.RawKey() |
||||
if k == nil { |
||||
return nil |
||||
} else if bytes.Equal(k, key) { |
||||
return it.RawValue() |
||||
} |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
type RangeLimitIterator struct { |
||||
it *Iterator |
||||
|
||||
r *Range |
||||
l *Limit |
||||
|
||||
step int |
||||
|
||||
//0 for IteratorForward, 1 for IteratorBackward
|
||||
direction uint8 |
||||
} |
||||
|
||||
func (it *RangeLimitIterator) Key() []byte { |
||||
return it.it.Key() |
||||
} |
||||
|
||||
func (it *RangeLimitIterator) Value() []byte { |
||||
return it.it.Value() |
||||
} |
||||
|
||||
func (it *RangeLimitIterator) RawKey() []byte { |
||||
return it.it.RawKey() |
||||
} |
||||
|
||||
func (it *RangeLimitIterator) RawValue() []byte { |
||||
return it.it.RawValue() |
||||
} |
||||
|
||||
func (it *RangeLimitIterator) BufKey(b []byte) []byte { |
||||
return it.it.BufKey(b) |
||||
} |
||||
|
||||
func (it *RangeLimitIterator) BufValue(b []byte) []byte { |
||||
return it.it.BufValue(b) |
||||
} |
||||
|
||||
func (it *RangeLimitIterator) Valid() bool { |
||||
if it.l.Offset < 0 { |
||||
return false |
||||
} else if !it.it.Valid() { |
||||
return false |
||||
} else if it.l.Count >= 0 && it.step >= it.l.Count { |
||||
return false |
||||
} |
||||
|
||||
if it.direction == IteratorForward { |
||||
if it.r.Max != nil { |
||||
r := bytes.Compare(it.it.RawKey(), it.r.Max) |
||||
if it.r.Type&RangeROpen > 0 { |
||||
return !(r >= 0) |
||||
} else { |
||||
return !(r > 0) |
||||
} |
||||
} |
||||
} else { |
||||
if it.r.Min != nil { |
||||
r := bytes.Compare(it.it.RawKey(), it.r.Min) |
||||
if it.r.Type&RangeLOpen > 0 { |
||||
return !(r <= 0) |
||||
} else { |
||||
return !(r < 0) |
||||
} |
||||
} |
||||
} |
||||
|
||||
return true |
||||
} |
||||
|
||||
func (it *RangeLimitIterator) Next() { |
||||
it.step++ |
||||
|
||||
if it.direction == IteratorForward { |
||||
it.it.Next() |
||||
} else { |
||||
it.it.Prev() |
||||
} |
||||
} |
||||
|
||||
func (it *RangeLimitIterator) Close() { |
||||
it.it.Close() |
||||
} |
||||
|
||||
func NewRangeLimitIterator(i *Iterator, r *Range, l *Limit) *RangeLimitIterator { |
||||
return rangeLimitIterator(i, r, l, IteratorForward) |
||||
} |
||||
|
||||
func NewRevRangeLimitIterator(i *Iterator, r *Range, l *Limit) *RangeLimitIterator { |
||||
return rangeLimitIterator(i, r, l, IteratorBackward) |
||||
} |
||||
|
||||
func NewRangeIterator(i *Iterator, r *Range) *RangeLimitIterator { |
||||
return rangeLimitIterator(i, r, &Limit{0, -1}, IteratorForward) |
||||
} |
||||
|
||||
func NewRevRangeIterator(i *Iterator, r *Range) *RangeLimitIterator { |
||||
return rangeLimitIterator(i, r, &Limit{0, -1}, IteratorBackward) |
||||
} |
||||
|
||||
func rangeLimitIterator(i *Iterator, r *Range, l *Limit, direction uint8) *RangeLimitIterator { |
||||
it := new(RangeLimitIterator) |
||||
|
||||
it.it = i |
||||
|
||||
it.r = r |
||||
it.l = l |
||||
it.direction = direction |
||||
|
||||
it.step = 0 |
||||
|
||||
if l.Offset < 0 { |
||||
return it |
||||
} |
||||
|
||||
if direction == IteratorForward { |
||||
if r.Min == nil { |
||||
it.it.SeekToFirst() |
||||
} else { |
||||
it.it.Seek(r.Min) |
||||
|
||||
if r.Type&RangeLOpen > 0 { |
||||
if it.it.Valid() && bytes.Equal(it.it.RawKey(), r.Min) { |
||||
it.it.Next() |
||||
} |
||||
} |
||||
} |
||||
} else { |
||||
if r.Max == nil { |
||||
it.it.SeekToLast() |
||||
} else { |
||||
it.it.Seek(r.Max) |
||||
|
||||
if !it.it.Valid() { |
||||
it.it.SeekToLast() |
||||
} else { |
||||
if !bytes.Equal(it.it.RawKey(), r.Max) { |
||||
it.it.Prev() |
||||
} |
||||
} |
||||
|
||||
if r.Type&RangeROpen > 0 { |
||||
if it.it.Valid() && bytes.Equal(it.it.RawKey(), r.Max) { |
||||
it.it.Prev() |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
for i := 0; i < l.Offset; i++ { |
||||
if it.it.Valid() { |
||||
if it.direction == IteratorForward { |
||||
it.it.Next() |
||||
} else { |
||||
it.it.Prev() |
||||
} |
||||
} |
||||
} |
||||
|
||||
return it |
||||
} |
||||
@ -0,0 +1,16 @@ |
||||
package store |
||||
|
||||
import ( |
||||
"github.com/lunny/nodb/store/driver" |
||||
) |
||||
|
||||
type Snapshot struct { |
||||
driver.ISnapshot |
||||
} |
||||
|
||||
func (s *Snapshot) NewIterator() *Iterator { |
||||
it := new(Iterator) |
||||
it.it = s.ISnapshot.NewIterator() |
||||
|
||||
return it |
||||
} |
||||
@ -0,0 +1,51 @@ |
||||
package store |
||||
|
||||
import ( |
||||
"fmt" |
||||
"os" |
||||
"path" |
||||
"github.com/lunny/nodb/config" |
||||
"github.com/lunny/nodb/store/driver" |
||||
|
||||
_ "github.com/lunny/nodb/store/goleveldb" |
||||
) |
||||
|
||||
func getStorePath(cfg *config.Config) string { |
||||
return path.Join(cfg.DataDir, fmt.Sprintf("%s_data", cfg.DBName)) |
||||
} |
||||
|
||||
func Open(cfg *config.Config) (*DB, error) { |
||||
s, err := driver.GetStore(cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
path := getStorePath(cfg) |
||||
|
||||
if err := os.MkdirAll(path, os.ModePerm); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
idb, err := s.Open(path, cfg) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
db := &DB{idb} |
||||
|
||||
return db, nil |
||||
} |
||||
|
||||
func Repair(cfg *config.Config) error { |
||||
s, err := driver.GetStore(cfg) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
path := getStorePath(cfg) |
||||
|
||||
return s.Repair(path, cfg) |
||||
} |
||||
|
||||
func init() { |
||||
} |
||||
@ -0,0 +1,42 @@ |
||||
package store |
||||
|
||||
import ( |
||||
"github.com/lunny/nodb/store/driver" |
||||
) |
||||
|
||||
type Tx struct { |
||||
driver.Tx |
||||
} |
||||
|
||||
func (tx *Tx) NewIterator() *Iterator { |
||||
it := new(Iterator) |
||||
it.it = tx.Tx.NewIterator() |
||||
|
||||
return it |
||||
} |
||||
|
||||
func (tx *Tx) NewWriteBatch() WriteBatch { |
||||
return tx.Tx.NewWriteBatch() |
||||
} |
||||
|
||||
func (tx *Tx) RangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator { |
||||
return NewRangeLimitIterator(tx.NewIterator(), &Range{min, max, rangeType}, &Limit{0, -1}) |
||||
} |
||||
|
||||
func (tx *Tx) RevRangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator { |
||||
return NewRevRangeLimitIterator(tx.NewIterator(), &Range{min, max, rangeType}, &Limit{0, -1}) |
||||
} |
||||
|
||||
//count < 0, unlimit.
|
||||
//
|
||||
//offset must >= 0, if < 0, will get nothing.
|
||||
func (tx *Tx) RangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *RangeLimitIterator { |
||||
return NewRangeLimitIterator(tx.NewIterator(), &Range{min, max, rangeType}, &Limit{offset, count}) |
||||
} |
||||
|
||||
//count < 0, unlimit.
|
||||
//
|
||||
//offset must >= 0, if < 0, will get nothing.
|
||||
func (tx *Tx) RevRangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *RangeLimitIterator { |
||||
return NewRevRangeLimitIterator(tx.NewIterator(), &Range{min, max, rangeType}, &Limit{offset, count}) |
||||
} |
||||
@ -0,0 +1,9 @@ |
||||
package store |
||||
|
||||
import ( |
||||
"github.com/lunny/nodb/store/driver" |
||||
) |
||||
|
||||
type WriteBatch interface { |
||||
driver.IWriteBatch |
||||
} |
||||
@ -0,0 +1,922 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
"errors" |
||||
"sort" |
||||
"time" |
||||
|
||||
"github.com/lunny/nodb/store" |
||||
) |
||||
|
||||
const ( |
||||
OPand uint8 = iota + 1 |
||||
OPor |
||||
OPxor |
||||
OPnot |
||||
) |
||||
|
||||
type BitPair struct { |
||||
Pos int32 |
||||
Val uint8 |
||||
} |
||||
|
||||
type segBitInfo struct { |
||||
Seq uint32 |
||||
Off uint32 |
||||
Val uint8 |
||||
} |
||||
|
||||
type segBitInfoArray []segBitInfo |
||||
|
||||
const ( |
||||
// byte
|
||||
segByteWidth uint32 = 9 |
||||
segByteSize uint32 = 1 << segByteWidth |
||||
|
||||
// bit
|
||||
segBitWidth uint32 = segByteWidth + 3 |
||||
segBitSize uint32 = segByteSize << 3 |
||||
|
||||
maxByteSize uint32 = 8 << 20 |
||||
maxSegCount uint32 = maxByteSize / segByteSize |
||||
|
||||
minSeq uint32 = 0 |
||||
maxSeq uint32 = uint32((maxByteSize << 3) - 1) |
||||
) |
||||
|
||||
var bitsInByte = [256]int32{0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, |
||||
4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, |
||||
3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, |
||||
5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, |
||||
3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, |
||||
5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, |
||||
2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, |
||||
4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, |
||||
3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, |
||||
5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, |
||||
6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, |
||||
6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8} |
||||
|
||||
var fillBits = [...]uint8{1, 3, 7, 15, 31, 63, 127, 255} |
||||
|
||||
var emptySegment []byte = make([]byte, segByteSize, segByteSize) |
||||
|
||||
var fillSegment []byte = func() []byte { |
||||
data := make([]byte, segByteSize, segByteSize) |
||||
for i := uint32(0); i < segByteSize; i++ { |
||||
data[i] = 0xff |
||||
} |
||||
return data |
||||
}() |
||||
|
||||
var errBinKey = errors.New("invalid bin key") |
||||
var errOffset = errors.New("invalid offset") |
||||
var errDuplicatePos = errors.New("duplicate bit pos") |
||||
|
||||
func getBit(sz []byte, offset uint32) uint8 { |
||||
index := offset >> 3 |
||||
if index >= uint32(len(sz)) { |
||||
return 0 // error("overflow")
|
||||
} |
||||
|
||||
offset -= index << 3 |
||||
return sz[index] >> offset & 1 |
||||
} |
||||
|
||||
func setBit(sz []byte, offset uint32, val uint8) bool { |
||||
if val != 1 && val != 0 { |
||||
return false // error("invalid val")
|
||||
} |
||||
|
||||
index := offset >> 3 |
||||
if index >= uint32(len(sz)) { |
||||
return false // error("overflow")
|
||||
} |
||||
|
||||
offset -= index << 3 |
||||
if sz[index]>>offset&1 != val { |
||||
sz[index] ^= (1 << offset) |
||||
} |
||||
return true |
||||
} |
||||
|
||||
func (datas segBitInfoArray) Len() int { |
||||
return len(datas) |
||||
} |
||||
|
||||
func (datas segBitInfoArray) Less(i, j int) bool { |
||||
res := (datas)[i].Seq < (datas)[j].Seq |
||||
if !res && (datas)[i].Seq == (datas)[j].Seq { |
||||
res = (datas)[i].Off < (datas)[j].Off |
||||
} |
||||
return res |
||||
} |
||||
|
||||
func (datas segBitInfoArray) Swap(i, j int) { |
||||
datas[i], datas[j] = datas[j], datas[i] |
||||
} |
||||
|
||||
func (db *DB) bEncodeMetaKey(key []byte) []byte { |
||||
mk := make([]byte, len(key)+2) |
||||
mk[0] = db.index |
||||
mk[1] = BitMetaType |
||||
|
||||
copy(mk[2:], key) |
||||
return mk |
||||
} |
||||
|
||||
func (db *DB) bDecodeMetaKey(bkey []byte) ([]byte, error) { |
||||
if len(bkey) < 2 || bkey[0] != db.index || bkey[1] != BitMetaType { |
||||
return nil, errBinKey |
||||
} |
||||
|
||||
return bkey[2:], nil |
||||
} |
||||
|
||||
func (db *DB) bEncodeBinKey(key []byte, seq uint32) []byte { |
||||
bk := make([]byte, len(key)+8) |
||||
|
||||
pos := 0 |
||||
bk[pos] = db.index |
||||
pos++ |
||||
bk[pos] = BitType |
||||
pos++ |
||||
|
||||
binary.BigEndian.PutUint16(bk[pos:], uint16(len(key))) |
||||
pos += 2 |
||||
|
||||
copy(bk[pos:], key) |
||||
pos += len(key) |
||||
|
||||
binary.BigEndian.PutUint32(bk[pos:], seq) |
||||
|
||||
return bk |
||||
} |
||||
|
||||
func (db *DB) bDecodeBinKey(bkey []byte) (key []byte, seq uint32, err error) { |
||||
if len(bkey) < 8 || bkey[0] != db.index { |
||||
err = errBinKey |
||||
return |
||||
} |
||||
|
||||
keyLen := binary.BigEndian.Uint16(bkey[2:4]) |
||||
if int(keyLen+8) != len(bkey) { |
||||
err = errBinKey |
||||
return |
||||
} |
||||
|
||||
key = bkey[4 : 4+keyLen] |
||||
seq = uint32(binary.BigEndian.Uint32(bkey[4+keyLen:])) |
||||
return |
||||
} |
||||
|
||||
func (db *DB) bCapByteSize(seq uint32, off uint32) uint32 { |
||||
var offByteSize uint32 = (off >> 3) + 1 |
||||
if offByteSize > segByteSize { |
||||
offByteSize = segByteSize |
||||
} |
||||
|
||||
return seq<<segByteWidth + offByteSize |
||||
} |
||||
|
||||
func (db *DB) bParseOffset(key []byte, offset int32) (seq uint32, off uint32, err error) { |
||||
if offset < 0 { |
||||
if tailSeq, tailOff, e := db.bGetMeta(key); e != nil { |
||||
err = e |
||||
return |
||||
} else if tailSeq >= 0 { |
||||
offset += int32((uint32(tailSeq)<<segBitWidth | uint32(tailOff)) + 1) |
||||
if offset < 0 { |
||||
err = errOffset |
||||
return |
||||
} |
||||
} |
||||
} |
||||
|
||||
off = uint32(offset) |
||||
|
||||
seq = off >> segBitWidth |
||||
off &= (segBitSize - 1) |
||||
return |
||||
} |
||||
|
||||
func (db *DB) bGetMeta(key []byte) (tailSeq int32, tailOff int32, err error) { |
||||
var v []byte |
||||
|
||||
mk := db.bEncodeMetaKey(key) |
||||
v, err = db.bucket.Get(mk) |
||||
if err != nil { |
||||
return |
||||
} |
||||
|
||||
if v != nil { |
||||
tailSeq = int32(binary.LittleEndian.Uint32(v[0:4])) |
||||
tailOff = int32(binary.LittleEndian.Uint32(v[4:8])) |
||||
} else { |
||||
tailSeq = -1 |
||||
tailOff = -1 |
||||
} |
||||
return |
||||
} |
||||
|
||||
func (db *DB) bSetMeta(t *batch, key []byte, tailSeq uint32, tailOff uint32) { |
||||
ek := db.bEncodeMetaKey(key) |
||||
|
||||
buf := make([]byte, 8) |
||||
binary.LittleEndian.PutUint32(buf[0:4], tailSeq) |
||||
binary.LittleEndian.PutUint32(buf[4:8], tailOff) |
||||
|
||||
t.Put(ek, buf) |
||||
return |
||||
} |
||||
|
||||
func (db *DB) bUpdateMeta(t *batch, key []byte, seq uint32, off uint32) (tailSeq uint32, tailOff uint32, err error) { |
||||
var tseq, toff int32 |
||||
var update bool = false |
||||
|
||||
if tseq, toff, err = db.bGetMeta(key); err != nil { |
||||
return |
||||
} else if tseq < 0 { |
||||
update = true |
||||
} else { |
||||
tailSeq = uint32(MaxInt32(tseq, 0)) |
||||
tailOff = uint32(MaxInt32(toff, 0)) |
||||
update = (seq > tailSeq || (seq == tailSeq && off > tailOff)) |
||||
} |
||||
|
||||
if update { |
||||
db.bSetMeta(t, key, seq, off) |
||||
tailSeq = seq |
||||
tailOff = off |
||||
} |
||||
return |
||||
} |
||||
|
||||
func (db *DB) bDelete(t *batch, key []byte) (drop int64) { |
||||
mk := db.bEncodeMetaKey(key) |
||||
t.Delete(mk) |
||||
|
||||
minKey := db.bEncodeBinKey(key, minSeq) |
||||
maxKey := db.bEncodeBinKey(key, maxSeq) |
||||
it := db.bucket.RangeIterator(minKey, maxKey, store.RangeClose) |
||||
for ; it.Valid(); it.Next() { |
||||
t.Delete(it.RawKey()) |
||||
drop++ |
||||
} |
||||
it.Close() |
||||
|
||||
return drop |
||||
} |
||||
|
||||
func (db *DB) bGetSegment(key []byte, seq uint32) ([]byte, []byte, error) { |
||||
bk := db.bEncodeBinKey(key, seq) |
||||
segment, err := db.bucket.Get(bk) |
||||
if err != nil { |
||||
return bk, nil, err |
||||
} |
||||
return bk, segment, nil |
||||
} |
||||
|
||||
func (db *DB) bAllocateSegment(key []byte, seq uint32) ([]byte, []byte, error) { |
||||
bk, segment, err := db.bGetSegment(key, seq) |
||||
if err == nil && segment == nil { |
||||
segment = make([]byte, segByteSize, segByteSize) |
||||
} |
||||
return bk, segment, err |
||||
} |
||||
|
||||
func (db *DB) bIterator(key []byte) *store.RangeLimitIterator { |
||||
sk := db.bEncodeBinKey(key, minSeq) |
||||
ek := db.bEncodeBinKey(key, maxSeq) |
||||
return db.bucket.RangeIterator(sk, ek, store.RangeClose) |
||||
} |
||||
|
||||
func (db *DB) bSegAnd(a []byte, b []byte, res *[]byte) { |
||||
if a == nil || b == nil { |
||||
*res = nil |
||||
return |
||||
} |
||||
|
||||
data := *res |
||||
if data == nil { |
||||
data = make([]byte, segByteSize, segByteSize) |
||||
*res = data |
||||
} |
||||
|
||||
for i := uint32(0); i < segByteSize; i++ { |
||||
data[i] = a[i] & b[i] |
||||
} |
||||
return |
||||
} |
||||
|
||||
func (db *DB) bSegOr(a []byte, b []byte, res *[]byte) { |
||||
if a == nil || b == nil { |
||||
if a == nil && b == nil { |
||||
*res = nil |
||||
} else if a == nil { |
||||
*res = b |
||||
} else { |
||||
*res = a |
||||
} |
||||
return |
||||
} |
||||
|
||||
data := *res |
||||
if data == nil { |
||||
data = make([]byte, segByteSize, segByteSize) |
||||
*res = data |
||||
} |
||||
|
||||
for i := uint32(0); i < segByteSize; i++ { |
||||
data[i] = a[i] | b[i] |
||||
} |
||||
return |
||||
} |
||||
|
||||
func (db *DB) bSegXor(a []byte, b []byte, res *[]byte) { |
||||
if a == nil && b == nil { |
||||
*res = fillSegment |
||||
return |
||||
} |
||||
|
||||
if a == nil { |
||||
a = emptySegment |
||||
} |
||||
|
||||
if b == nil { |
||||
b = emptySegment |
||||
} |
||||
|
||||
data := *res |
||||
if data == nil { |
||||
data = make([]byte, segByteSize, segByteSize) |
||||
*res = data |
||||
} |
||||
|
||||
for i := uint32(0); i < segByteSize; i++ { |
||||
data[i] = a[i] ^ b[i] |
||||
} |
||||
|
||||
return |
||||
} |
||||
|
||||
func (db *DB) bExpireAt(key []byte, when int64) (int64, error) { |
||||
t := db.binBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
if seq, _, err := db.bGetMeta(key); err != nil || seq < 0 { |
||||
return 0, err |
||||
} else { |
||||
db.expireAt(t, BitType, key, when) |
||||
if err := t.Commit(); err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
return 1, nil |
||||
} |
||||
|
||||
func (db *DB) bCountByte(val byte, soff uint32, eoff uint32) int32 { |
||||
if soff > eoff { |
||||
soff, eoff = eoff, soff |
||||
} |
||||
|
||||
mask := uint8(0) |
||||
if soff > 0 { |
||||
mask |= fillBits[soff-1] |
||||
} |
||||
if eoff < 7 { |
||||
mask |= (fillBits[7] ^ fillBits[eoff]) |
||||
} |
||||
mask = fillBits[7] ^ mask |
||||
|
||||
return bitsInByte[val&mask] |
||||
} |
||||
|
||||
func (db *DB) bCountSeg(key []byte, seq uint32, soff uint32, eoff uint32) (cnt int32, err error) { |
||||
if soff >= segBitSize || soff < 0 || |
||||
eoff >= segBitSize || eoff < 0 { |
||||
return |
||||
} |
||||
|
||||
var segment []byte |
||||
if _, segment, err = db.bGetSegment(key, seq); err != nil { |
||||
return |
||||
} |
||||
|
||||
if segment == nil { |
||||
return |
||||
} |
||||
|
||||
if soff > eoff { |
||||
soff, eoff = eoff, soff |
||||
} |
||||
|
||||
headIdx := int(soff >> 3) |
||||
endIdx := int(eoff >> 3) |
||||
sByteOff := soff - ((soff >> 3) << 3) |
||||
eByteOff := eoff - ((eoff >> 3) << 3) |
||||
|
||||
if headIdx == endIdx { |
||||
cnt = db.bCountByte(segment[headIdx], sByteOff, eByteOff) |
||||
} else { |
||||
cnt = db.bCountByte(segment[headIdx], sByteOff, 7) + |
||||
db.bCountByte(segment[endIdx], 0, eByteOff) |
||||
} |
||||
|
||||
// sum up following bytes
|
||||
for idx, end := headIdx+1, endIdx-1; idx <= end; idx += 1 { |
||||
cnt += bitsInByte[segment[idx]] |
||||
if idx == end { |
||||
break |
||||
} |
||||
} |
||||
|
||||
return |
||||
} |
||||
|
||||
func (db *DB) BGet(key []byte) (data []byte, err error) { |
||||
if err = checkKeySize(key); err != nil { |
||||
return |
||||
} |
||||
|
||||
var ts, to int32 |
||||
if ts, to, err = db.bGetMeta(key); err != nil || ts < 0 { |
||||
return |
||||
} |
||||
|
||||
var tailSeq, tailOff = uint32(ts), uint32(to) |
||||
var capByteSize uint32 = db.bCapByteSize(tailSeq, tailOff) |
||||
data = make([]byte, capByteSize, capByteSize) |
||||
|
||||
minKey := db.bEncodeBinKey(key, minSeq) |
||||
maxKey := db.bEncodeBinKey(key, tailSeq) |
||||
it := db.bucket.RangeIterator(minKey, maxKey, store.RangeClose) |
||||
|
||||
var seq, s, e uint32 |
||||
for ; it.Valid(); it.Next() { |
||||
if _, seq, err = db.bDecodeBinKey(it.RawKey()); err != nil { |
||||
data = nil |
||||
break |
||||
} |
||||
|
||||
s = seq << segByteWidth |
||||
e = MinUInt32(s+segByteSize, capByteSize) |
||||
copy(data[s:e], it.RawValue()) |
||||
} |
||||
it.Close() |
||||
|
||||
return |
||||
} |
||||
|
||||
func (db *DB) BDelete(key []byte) (drop int64, err error) { |
||||
if err = checkKeySize(key); err != nil { |
||||
return |
||||
} |
||||
|
||||
t := db.binBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
drop = db.bDelete(t, key) |
||||
db.rmExpire(t, BitType, key) |
||||
|
||||
err = t.Commit() |
||||
return |
||||
} |
||||
|
||||
func (db *DB) BSetBit(key []byte, offset int32, val uint8) (ori uint8, err error) { |
||||
if err = checkKeySize(key); err != nil { |
||||
return |
||||
} |
||||
|
||||
// todo : check offset
|
||||
var seq, off uint32 |
||||
if seq, off, err = db.bParseOffset(key, offset); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
var bk, segment []byte |
||||
if bk, segment, err = db.bAllocateSegment(key, seq); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
if segment != nil { |
||||
ori = getBit(segment, off) |
||||
if setBit(segment, off, val) { |
||||
t := db.binBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
t.Put(bk, segment) |
||||
if _, _, e := db.bUpdateMeta(t, key, seq, off); e != nil { |
||||
err = e |
||||
return |
||||
} |
||||
|
||||
err = t.Commit() |
||||
} |
||||
} |
||||
|
||||
return |
||||
} |
||||
|
||||
func (db *DB) BMSetBit(key []byte, args ...BitPair) (place int64, err error) { |
||||
if err = checkKeySize(key); err != nil { |
||||
return |
||||
} |
||||
|
||||
// (ps : so as to aviod wasting memory copy while calling db.Get() and batch.Put(),
|
||||
// here we sequence the params by pos, so that we can merge the execution of
|
||||
// diff pos setting which targets on the same segment respectively. )
|
||||
|
||||
// #1 : sequence request data
|
||||
var argCnt = len(args) |
||||
var bitInfos segBitInfoArray = make(segBitInfoArray, argCnt) |
||||
var seq, off uint32 |
||||
|
||||
for i, info := range args { |
||||
if seq, off, err = db.bParseOffset(key, info.Pos); err != nil { |
||||
return |
||||
} |
||||
|
||||
bitInfos[i].Seq = seq |
||||
bitInfos[i].Off = off |
||||
bitInfos[i].Val = info.Val |
||||
} |
||||
|
||||
sort.Sort(bitInfos) |
||||
|
||||
for i := 1; i < argCnt; i++ { |
||||
if bitInfos[i].Seq == bitInfos[i-1].Seq && bitInfos[i].Off == bitInfos[i-1].Off { |
||||
return 0, errDuplicatePos |
||||
} |
||||
} |
||||
|
||||
// #2 : execute bit set in order
|
||||
t := db.binBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
var curBinKey, curSeg []byte |
||||
var curSeq, maxSeq, maxOff uint32 |
||||
|
||||
for _, info := range bitInfos { |
||||
if curSeg != nil && info.Seq != curSeq { |
||||
t.Put(curBinKey, curSeg) |
||||
curSeg = nil |
||||
} |
||||
|
||||
if curSeg == nil { |
||||
curSeq = info.Seq |
||||
if curBinKey, curSeg, err = db.bAllocateSegment(key, info.Seq); err != nil { |
||||
return |
||||
} |
||||
|
||||
if curSeg == nil { |
||||
continue |
||||
} |
||||
} |
||||
|
||||
if setBit(curSeg, info.Off, info.Val) { |
||||
maxSeq = info.Seq |
||||
maxOff = info.Off |
||||
place++ |
||||
} |
||||
} |
||||
|
||||
if curSeg != nil { |
||||
t.Put(curBinKey, curSeg) |
||||
} |
||||
|
||||
// finally, update meta
|
||||
if place > 0 { |
||||
if _, _, err = db.bUpdateMeta(t, key, maxSeq, maxOff); err != nil { |
||||
return |
||||
} |
||||
|
||||
err = t.Commit() |
||||
} |
||||
|
||||
return |
||||
} |
||||
|
||||
func (db *DB) BGetBit(key []byte, offset int32) (uint8, error) { |
||||
if seq, off, err := db.bParseOffset(key, offset); err != nil { |
||||
return 0, err |
||||
} else { |
||||
_, segment, err := db.bGetSegment(key, seq) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
if segment == nil { |
||||
return 0, nil |
||||
} else { |
||||
return getBit(segment, off), nil |
||||
} |
||||
} |
||||
} |
||||
|
||||
// func (db *DB) BGetRange(key []byte, start int32, end int32) ([]byte, error) {
|
||||
// section := make([]byte)
|
||||
|
||||
// return
|
||||
// }
|
||||
|
||||
func (db *DB) BCount(key []byte, start int32, end int32) (cnt int32, err error) { |
||||
var sseq, soff uint32 |
||||
if sseq, soff, err = db.bParseOffset(key, start); err != nil { |
||||
return |
||||
} |
||||
|
||||
var eseq, eoff uint32 |
||||
if eseq, eoff, err = db.bParseOffset(key, end); err != nil { |
||||
return |
||||
} |
||||
|
||||
if sseq > eseq || (sseq == eseq && soff > eoff) { |
||||
sseq, eseq = eseq, sseq |
||||
soff, eoff = eoff, soff |
||||
} |
||||
|
||||
var segCnt int32 |
||||
if eseq == sseq { |
||||
if segCnt, err = db.bCountSeg(key, sseq, soff, eoff); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
cnt = segCnt |
||||
|
||||
} else { |
||||
if segCnt, err = db.bCountSeg(key, sseq, soff, segBitSize-1); err != nil { |
||||
return 0, err |
||||
} else { |
||||
cnt += segCnt |
||||
} |
||||
|
||||
if segCnt, err = db.bCountSeg(key, eseq, 0, eoff); err != nil { |
||||
return 0, err |
||||
} else { |
||||
cnt += segCnt |
||||
} |
||||
} |
||||
|
||||
// middle segs
|
||||
var segment []byte |
||||
skey := db.bEncodeBinKey(key, sseq) |
||||
ekey := db.bEncodeBinKey(key, eseq) |
||||
|
||||
it := db.bucket.RangeIterator(skey, ekey, store.RangeOpen) |
||||
for ; it.Valid(); it.Next() { |
||||
segment = it.RawValue() |
||||
for _, bt := range segment { |
||||
cnt += bitsInByte[bt] |
||||
} |
||||
} |
||||
it.Close() |
||||
|
||||
return |
||||
} |
||||
|
||||
func (db *DB) BTail(key []byte) (int32, error) { |
||||
// effective length of data, the highest bit-pos set in history
|
||||
tailSeq, tailOff, err := db.bGetMeta(key) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
tail := int32(-1) |
||||
if tailSeq >= 0 { |
||||
tail = int32(uint32(tailSeq)<<segBitWidth | uint32(tailOff)) |
||||
} |
||||
|
||||
return tail, nil |
||||
} |
||||
|
||||
func (db *DB) BOperation(op uint8, dstkey []byte, srckeys ...[]byte) (blen int32, err error) { |
||||
// blen -
|
||||
// the total bit size of data stored in destination key,
|
||||
// that is equal to the size of the longest input string.
|
||||
|
||||
var exeOp func([]byte, []byte, *[]byte) |
||||
switch op { |
||||
case OPand: |
||||
exeOp = db.bSegAnd |
||||
case OPor: |
||||
exeOp = db.bSegOr |
||||
case OPxor, OPnot: |
||||
exeOp = db.bSegXor |
||||
default: |
||||
return |
||||
} |
||||
|
||||
if dstkey == nil || srckeys == nil { |
||||
return |
||||
} |
||||
|
||||
t := db.binBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
var srcKseq, srcKoff int32 |
||||
var seq, off, maxDstSeq, maxDstOff uint32 |
||||
|
||||
var keyNum int = len(srckeys) |
||||
var validKeyNum int |
||||
for i := 0; i < keyNum; i++ { |
||||
if srcKseq, srcKoff, err = db.bGetMeta(srckeys[i]); err != nil { |
||||
return |
||||
} else if srcKseq < 0 { |
||||
srckeys[i] = nil |
||||
continue |
||||
} |
||||
|
||||
validKeyNum++ |
||||
|
||||
seq = uint32(srcKseq) |
||||
off = uint32(srcKoff) |
||||
if seq > maxDstSeq || (seq == maxDstSeq && off > maxDstOff) { |
||||
maxDstSeq = seq |
||||
maxDstOff = off |
||||
} |
||||
} |
||||
|
||||
if (op == OPnot && validKeyNum != 1) || |
||||
(op != OPnot && validKeyNum < 2) { |
||||
return // with not enough existing source key
|
||||
} |
||||
|
||||
var srcIdx int |
||||
for srcIdx = 0; srcIdx < keyNum; srcIdx++ { |
||||
if srckeys[srcIdx] != nil { |
||||
break |
||||
} |
||||
} |
||||
|
||||
// init - data
|
||||
var segments = make([][]byte, maxDstSeq+1) |
||||
|
||||
if op == OPnot { |
||||
// ps :
|
||||
// ( ~num == num ^ 0x11111111 )
|
||||
// we init the result segments with all bit set,
|
||||
// then we can calculate through the way of 'xor'.
|
||||
|
||||
// ahead segments bin format : 1111 ... 1111
|
||||
for i := uint32(0); i < maxDstSeq; i++ { |
||||
segments[i] = fillSegment |
||||
} |
||||
|
||||
// last segment bin format : 1111..1100..0000
|
||||
var tailSeg = make([]byte, segByteSize, segByteSize) |
||||
var fillByte = fillBits[7] |
||||
var tailSegLen = db.bCapByteSize(uint32(0), maxDstOff) |
||||
for i := uint32(0); i < tailSegLen-1; i++ { |
||||
tailSeg[i] = fillByte |
||||
} |
||||
tailSeg[tailSegLen-1] = fillBits[maxDstOff-(tailSegLen-1)<<3] |
||||
segments[maxDstSeq] = tailSeg |
||||
|
||||
} else { |
||||
// ps : init segments by data corresponding to the 1st valid source key
|
||||
it := db.bIterator(srckeys[srcIdx]) |
||||
for ; it.Valid(); it.Next() { |
||||
if _, seq, err = db.bDecodeBinKey(it.RawKey()); err != nil { |
||||
// to do ...
|
||||
it.Close() |
||||
return |
||||
} |
||||
segments[seq] = it.Value() |
||||
} |
||||
it.Close() |
||||
srcIdx++ |
||||
} |
||||
|
||||
// operation with following keys
|
||||
var res []byte |
||||
for i := srcIdx; i < keyNum; i++ { |
||||
if srckeys[i] == nil { |
||||
continue |
||||
} |
||||
|
||||
it := db.bIterator(srckeys[i]) |
||||
for idx, end := uint32(0), false; !end; it.Next() { |
||||
end = !it.Valid() |
||||
if !end { |
||||
if _, seq, err = db.bDecodeBinKey(it.RawKey()); err != nil { |
||||
// to do ...
|
||||
it.Close() |
||||
return |
||||
} |
||||
} else { |
||||
seq = maxDstSeq + 1 |
||||
} |
||||
|
||||
// todo :
|
||||
// operation 'and' can be optimize here :
|
||||
// if seq > max_segments_idx, this loop can be break,
|
||||
// which can avoid cost from Key() and bDecodeBinKey()
|
||||
|
||||
for ; idx < seq; idx++ { |
||||
res = nil |
||||
exeOp(segments[idx], nil, &res) |
||||
segments[idx] = res |
||||
} |
||||
|
||||
if !end { |
||||
res = it.Value() |
||||
exeOp(segments[seq], res, &res) |
||||
segments[seq] = res |
||||
idx++ |
||||
} |
||||
} |
||||
it.Close() |
||||
} |
||||
|
||||
// clear the old data in case
|
||||
db.bDelete(t, dstkey) |
||||
db.rmExpire(t, BitType, dstkey) |
||||
|
||||
// set data
|
||||
db.bSetMeta(t, dstkey, maxDstSeq, maxDstOff) |
||||
|
||||
var bk []byte |
||||
for seq, segt := range segments { |
||||
if segt != nil { |
||||
bk = db.bEncodeBinKey(dstkey, uint32(seq)) |
||||
t.Put(bk, segt) |
||||
} |
||||
} |
||||
|
||||
err = t.Commit() |
||||
if err == nil { |
||||
// blen = int32(db.bCapByteSize(maxDstOff, maxDstOff))
|
||||
blen = int32(maxDstSeq<<segBitWidth | maxDstOff + 1) |
||||
} |
||||
|
||||
return |
||||
} |
||||
|
||||
func (db *DB) BExpire(key []byte, duration int64) (int64, error) { |
||||
if duration <= 0 { |
||||
return 0, errExpireValue |
||||
} |
||||
|
||||
if err := checkKeySize(key); err != nil { |
||||
return -1, err |
||||
} |
||||
|
||||
return db.bExpireAt(key, time.Now().Unix()+duration) |
||||
} |
||||
|
||||
func (db *DB) BExpireAt(key []byte, when int64) (int64, error) { |
||||
if when <= time.Now().Unix() { |
||||
return 0, errExpireValue |
||||
} |
||||
|
||||
if err := checkKeySize(key); err != nil { |
||||
return -1, err |
||||
} |
||||
|
||||
return db.bExpireAt(key, when) |
||||
} |
||||
|
||||
func (db *DB) BTTL(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return -1, err |
||||
} |
||||
|
||||
return db.ttl(BitType, key) |
||||
} |
||||
|
||||
func (db *DB) BPersist(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
t := db.binBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
n, err := db.rmExpire(t, BitType, key) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
err = t.Commit() |
||||
return n, err |
||||
} |
||||
|
||||
func (db *DB) BScan(key []byte, count int, inclusive bool, match string) ([][]byte, error) { |
||||
return db.scan(BitMetaType, key, count, inclusive, match) |
||||
} |
||||
|
||||
func (db *DB) bFlush() (drop int64, err error) { |
||||
t := db.binBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
return db.flushType(t, BitType) |
||||
} |
||||
@ -0,0 +1,509 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
"errors" |
||||
"time" |
||||
|
||||
"github.com/lunny/nodb/store" |
||||
) |
||||
|
||||
type FVPair struct { |
||||
Field []byte |
||||
Value []byte |
||||
} |
||||
|
||||
var errHashKey = errors.New("invalid hash key") |
||||
var errHSizeKey = errors.New("invalid hsize key") |
||||
|
||||
const ( |
||||
hashStartSep byte = ':' |
||||
hashStopSep byte = hashStartSep + 1 |
||||
) |
||||
|
||||
func checkHashKFSize(key []byte, field []byte) error { |
||||
if len(key) > MaxKeySize || len(key) == 0 { |
||||
return errKeySize |
||||
} else if len(field) > MaxHashFieldSize || len(field) == 0 { |
||||
return errHashFieldSize |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (db *DB) hEncodeSizeKey(key []byte) []byte { |
||||
buf := make([]byte, len(key)+2) |
||||
|
||||
buf[0] = db.index |
||||
buf[1] = HSizeType |
||||
|
||||
copy(buf[2:], key) |
||||
return buf |
||||
} |
||||
|
||||
func (db *DB) hDecodeSizeKey(ek []byte) ([]byte, error) { |
||||
if len(ek) < 2 || ek[0] != db.index || ek[1] != HSizeType { |
||||
return nil, errHSizeKey |
||||
} |
||||
|
||||
return ek[2:], nil |
||||
} |
||||
|
||||
func (db *DB) hEncodeHashKey(key []byte, field []byte) []byte { |
||||
buf := make([]byte, len(key)+len(field)+1+1+2+1) |
||||
|
||||
pos := 0 |
||||
buf[pos] = db.index |
||||
pos++ |
||||
buf[pos] = HashType |
||||
pos++ |
||||
|
||||
binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) |
||||
pos += 2 |
||||
|
||||
copy(buf[pos:], key) |
||||
pos += len(key) |
||||
|
||||
buf[pos] = hashStartSep |
||||
pos++ |
||||
copy(buf[pos:], field) |
||||
|
||||
return buf |
||||
} |
||||
|
||||
func (db *DB) hDecodeHashKey(ek []byte) ([]byte, []byte, error) { |
||||
if len(ek) < 5 || ek[0] != db.index || ek[1] != HashType { |
||||
return nil, nil, errHashKey |
||||
} |
||||
|
||||
pos := 2 |
||||
keyLen := int(binary.BigEndian.Uint16(ek[pos:])) |
||||
pos += 2 |
||||
|
||||
if keyLen+5 > len(ek) { |
||||
return nil, nil, errHashKey |
||||
} |
||||
|
||||
key := ek[pos : pos+keyLen] |
||||
pos += keyLen |
||||
|
||||
if ek[pos] != hashStartSep { |
||||
return nil, nil, errHashKey |
||||
} |
||||
|
||||
pos++ |
||||
field := ek[pos:] |
||||
return key, field, nil |
||||
} |
||||
|
||||
func (db *DB) hEncodeStartKey(key []byte) []byte { |
||||
return db.hEncodeHashKey(key, nil) |
||||
} |
||||
|
||||
func (db *DB) hEncodeStopKey(key []byte) []byte { |
||||
k := db.hEncodeHashKey(key, nil) |
||||
|
||||
k[len(k)-1] = hashStopSep |
||||
|
||||
return k |
||||
} |
||||
|
||||
func (db *DB) hSetItem(key []byte, field []byte, value []byte) (int64, error) { |
||||
t := db.hashBatch |
||||
|
||||
ek := db.hEncodeHashKey(key, field) |
||||
|
||||
var n int64 = 1 |
||||
if v, _ := db.bucket.Get(ek); v != nil { |
||||
n = 0 |
||||
} else { |
||||
if _, err := db.hIncrSize(key, 1); err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
|
||||
t.Put(ek, value) |
||||
return n, nil |
||||
} |
||||
|
||||
// ps : here just focus on deleting the hash data,
|
||||
// any other likes expire is ignore.
|
||||
func (db *DB) hDelete(t *batch, key []byte) int64 { |
||||
sk := db.hEncodeSizeKey(key) |
||||
start := db.hEncodeStartKey(key) |
||||
stop := db.hEncodeStopKey(key) |
||||
|
||||
var num int64 = 0 |
||||
it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) |
||||
for ; it.Valid(); it.Next() { |
||||
t.Delete(it.Key()) |
||||
num++ |
||||
} |
||||
it.Close() |
||||
|
||||
t.Delete(sk) |
||||
return num |
||||
} |
||||
|
||||
func (db *DB) hExpireAt(key []byte, when int64) (int64, error) { |
||||
t := db.hashBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
if hlen, err := db.HLen(key); err != nil || hlen == 0 { |
||||
return 0, err |
||||
} else { |
||||
db.expireAt(t, HashType, key, when) |
||||
if err := t.Commit(); err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
return 1, nil |
||||
} |
||||
|
||||
func (db *DB) HLen(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
return Int64(db.bucket.Get(db.hEncodeSizeKey(key))) |
||||
} |
||||
|
||||
func (db *DB) HSet(key []byte, field []byte, value []byte) (int64, error) { |
||||
if err := checkHashKFSize(key, field); err != nil { |
||||
return 0, err |
||||
} else if err := checkValueSize(value); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
t := db.hashBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
n, err := db.hSetItem(key, field, value) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
//todo add binlog
|
||||
|
||||
err = t.Commit() |
||||
return n, err |
||||
} |
||||
|
||||
func (db *DB) HGet(key []byte, field []byte) ([]byte, error) { |
||||
if err := checkHashKFSize(key, field); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return db.bucket.Get(db.hEncodeHashKey(key, field)) |
||||
} |
||||
|
||||
func (db *DB) HMset(key []byte, args ...FVPair) error { |
||||
t := db.hashBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
var err error |
||||
var ek []byte |
||||
var num int64 = 0 |
||||
for i := 0; i < len(args); i++ { |
||||
if err := checkHashKFSize(key, args[i].Field); err != nil { |
||||
return err |
||||
} else if err := checkValueSize(args[i].Value); err != nil { |
||||
return err |
||||
} |
||||
|
||||
ek = db.hEncodeHashKey(key, args[i].Field) |
||||
|
||||
if v, err := db.bucket.Get(ek); err != nil { |
||||
return err |
||||
} else if v == nil { |
||||
num++ |
||||
} |
||||
|
||||
t.Put(ek, args[i].Value) |
||||
} |
||||
|
||||
if _, err = db.hIncrSize(key, num); err != nil { |
||||
return err |
||||
} |
||||
|
||||
//todo add binglog
|
||||
err = t.Commit() |
||||
return err |
||||
} |
||||
|
||||
func (db *DB) HMget(key []byte, args ...[]byte) ([][]byte, error) { |
||||
var ek []byte |
||||
|
||||
it := db.bucket.NewIterator() |
||||
defer it.Close() |
||||
|
||||
r := make([][]byte, len(args)) |
||||
for i := 0; i < len(args); i++ { |
||||
if err := checkHashKFSize(key, args[i]); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
ek = db.hEncodeHashKey(key, args[i]) |
||||
|
||||
r[i] = it.Find(ek) |
||||
} |
||||
|
||||
return r, nil |
||||
} |
||||
|
||||
func (db *DB) HDel(key []byte, args ...[]byte) (int64, error) { |
||||
t := db.hashBatch |
||||
|
||||
var ek []byte |
||||
var v []byte |
||||
var err error |
||||
|
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
it := db.bucket.NewIterator() |
||||
defer it.Close() |
||||
|
||||
var num int64 = 0 |
||||
for i := 0; i < len(args); i++ { |
||||
if err := checkHashKFSize(key, args[i]); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
ek = db.hEncodeHashKey(key, args[i]) |
||||
|
||||
v = it.RawFind(ek) |
||||
if v == nil { |
||||
continue |
||||
} else { |
||||
num++ |
||||
t.Delete(ek) |
||||
} |
||||
} |
||||
|
||||
if _, err = db.hIncrSize(key, -num); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
err = t.Commit() |
||||
|
||||
return num, err |
||||
} |
||||
|
||||
func (db *DB) hIncrSize(key []byte, delta int64) (int64, error) { |
||||
t := db.hashBatch |
||||
sk := db.hEncodeSizeKey(key) |
||||
|
||||
var err error |
||||
var size int64 = 0 |
||||
if size, err = Int64(db.bucket.Get(sk)); err != nil { |
||||
return 0, err |
||||
} else { |
||||
size += delta |
||||
if size <= 0 { |
||||
size = 0 |
||||
t.Delete(sk) |
||||
db.rmExpire(t, HashType, key) |
||||
} else { |
||||
t.Put(sk, PutInt64(size)) |
||||
} |
||||
} |
||||
|
||||
return size, nil |
||||
} |
||||
|
||||
func (db *DB) HIncrBy(key []byte, field []byte, delta int64) (int64, error) { |
||||
if err := checkHashKFSize(key, field); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
t := db.hashBatch |
||||
var ek []byte |
||||
var err error |
||||
|
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
ek = db.hEncodeHashKey(key, field) |
||||
|
||||
var n int64 = 0 |
||||
if n, err = StrInt64(db.bucket.Get(ek)); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
n += delta |
||||
|
||||
_, err = db.hSetItem(key, field, StrPutInt64(n)) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
err = t.Commit() |
||||
|
||||
return n, err |
||||
} |
||||
|
||||
func (db *DB) HGetAll(key []byte) ([]FVPair, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
start := db.hEncodeStartKey(key) |
||||
stop := db.hEncodeStopKey(key) |
||||
|
||||
v := make([]FVPair, 0, 16) |
||||
|
||||
it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) |
||||
for ; it.Valid(); it.Next() { |
||||
_, f, err := db.hDecodeHashKey(it.Key()) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
v = append(v, FVPair{Field: f, Value: it.Value()}) |
||||
} |
||||
|
||||
it.Close() |
||||
|
||||
return v, nil |
||||
} |
||||
|
||||
func (db *DB) HKeys(key []byte) ([][]byte, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
start := db.hEncodeStartKey(key) |
||||
stop := db.hEncodeStopKey(key) |
||||
|
||||
v := make([][]byte, 0, 16) |
||||
|
||||
it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) |
||||
for ; it.Valid(); it.Next() { |
||||
_, f, err := db.hDecodeHashKey(it.Key()) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
v = append(v, f) |
||||
} |
||||
|
||||
it.Close() |
||||
|
||||
return v, nil |
||||
} |
||||
|
||||
func (db *DB) HValues(key []byte) ([][]byte, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
start := db.hEncodeStartKey(key) |
||||
stop := db.hEncodeStopKey(key) |
||||
|
||||
v := make([][]byte, 0, 16) |
||||
|
||||
it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) |
||||
for ; it.Valid(); it.Next() { |
||||
_, _, err := db.hDecodeHashKey(it.Key()) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
v = append(v, it.Value()) |
||||
} |
||||
|
||||
it.Close() |
||||
|
||||
return v, nil |
||||
} |
||||
|
||||
func (db *DB) HClear(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
t := db.hashBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
num := db.hDelete(t, key) |
||||
db.rmExpire(t, HashType, key) |
||||
|
||||
err := t.Commit() |
||||
return num, err |
||||
} |
||||
|
||||
func (db *DB) HMclear(keys ...[]byte) (int64, error) { |
||||
t := db.hashBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
for _, key := range keys { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
db.hDelete(t, key) |
||||
db.rmExpire(t, HashType, key) |
||||
} |
||||
|
||||
err := t.Commit() |
||||
return int64(len(keys)), err |
||||
} |
||||
|
||||
func (db *DB) hFlush() (drop int64, err error) { |
||||
t := db.hashBatch |
||||
|
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
return db.flushType(t, HashType) |
||||
} |
||||
|
||||
func (db *DB) HScan(key []byte, count int, inclusive bool, match string) ([][]byte, error) { |
||||
return db.scan(HSizeType, key, count, inclusive, match) |
||||
} |
||||
|
||||
func (db *DB) HExpire(key []byte, duration int64) (int64, error) { |
||||
if duration <= 0 { |
||||
return 0, errExpireValue |
||||
} |
||||
|
||||
return db.hExpireAt(key, time.Now().Unix()+duration) |
||||
} |
||||
|
||||
func (db *DB) HExpireAt(key []byte, when int64) (int64, error) { |
||||
if when <= time.Now().Unix() { |
||||
return 0, errExpireValue |
||||
} |
||||
|
||||
return db.hExpireAt(key, when) |
||||
} |
||||
|
||||
func (db *DB) HTTL(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return -1, err |
||||
} |
||||
|
||||
return db.ttl(HashType, key) |
||||
} |
||||
|
||||
func (db *DB) HPersist(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
t := db.hashBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
n, err := db.rmExpire(t, HashType, key) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
err = t.Commit() |
||||
return n, err |
||||
} |
||||
@ -0,0 +1,387 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"errors" |
||||
"time" |
||||
) |
||||
|
||||
type KVPair struct { |
||||
Key []byte |
||||
Value []byte |
||||
} |
||||
|
||||
var errKVKey = errors.New("invalid encode kv key") |
||||
|
||||
func checkKeySize(key []byte) error { |
||||
if len(key) > MaxKeySize || len(key) == 0 { |
||||
return errKeySize |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func checkValueSize(value []byte) error { |
||||
if len(value) > MaxValueSize { |
||||
return errValueSize |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (db *DB) encodeKVKey(key []byte) []byte { |
||||
ek := make([]byte, len(key)+2) |
||||
ek[0] = db.index |
||||
ek[1] = KVType |
||||
copy(ek[2:], key) |
||||
return ek |
||||
} |
||||
|
||||
func (db *DB) decodeKVKey(ek []byte) ([]byte, error) { |
||||
if len(ek) < 2 || ek[0] != db.index || ek[1] != KVType { |
||||
return nil, errKVKey |
||||
} |
||||
|
||||
return ek[2:], nil |
||||
} |
||||
|
||||
func (db *DB) encodeKVMinKey() []byte { |
||||
ek := db.encodeKVKey(nil) |
||||
return ek |
||||
} |
||||
|
||||
func (db *DB) encodeKVMaxKey() []byte { |
||||
ek := db.encodeKVKey(nil) |
||||
ek[len(ek)-1] = KVType + 1 |
||||
return ek |
||||
} |
||||
|
||||
func (db *DB) incr(key []byte, delta int64) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
var err error |
||||
key = db.encodeKVKey(key) |
||||
|
||||
t := db.kvBatch |
||||
|
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
var n int64 |
||||
n, err = StrInt64(db.bucket.Get(key)) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
n += delta |
||||
|
||||
t.Put(key, StrPutInt64(n)) |
||||
|
||||
//todo binlog
|
||||
|
||||
err = t.Commit() |
||||
return n, err |
||||
} |
||||
|
||||
// ps : here just focus on deleting the key-value data,
|
||||
// any other likes expire is ignore.
|
||||
func (db *DB) delete(t *batch, key []byte) int64 { |
||||
key = db.encodeKVKey(key) |
||||
t.Delete(key) |
||||
return 1 |
||||
} |
||||
|
||||
func (db *DB) setExpireAt(key []byte, when int64) (int64, error) { |
||||
t := db.kvBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
if exist, err := db.Exists(key); err != nil || exist == 0 { |
||||
return 0, err |
||||
} else { |
||||
db.expireAt(t, KVType, key, when) |
||||
if err := t.Commit(); err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
return 1, nil |
||||
} |
||||
|
||||
func (db *DB) Decr(key []byte) (int64, error) { |
||||
return db.incr(key, -1) |
||||
} |
||||
|
||||
func (db *DB) DecrBy(key []byte, decrement int64) (int64, error) { |
||||
return db.incr(key, -decrement) |
||||
} |
||||
|
||||
func (db *DB) Del(keys ...[]byte) (int64, error) { |
||||
if len(keys) == 0 { |
||||
return 0, nil |
||||
} |
||||
|
||||
codedKeys := make([][]byte, len(keys)) |
||||
for i, k := range keys { |
||||
codedKeys[i] = db.encodeKVKey(k) |
||||
} |
||||
|
||||
t := db.kvBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
for i, k := range keys { |
||||
t.Delete(codedKeys[i]) |
||||
db.rmExpire(t, KVType, k) |
||||
} |
||||
|
||||
err := t.Commit() |
||||
return int64(len(keys)), err |
||||
} |
||||
|
||||
func (db *DB) Exists(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
var err error |
||||
key = db.encodeKVKey(key) |
||||
|
||||
var v []byte |
||||
v, err = db.bucket.Get(key) |
||||
if v != nil && err == nil { |
||||
return 1, nil |
||||
} |
||||
|
||||
return 0, err |
||||
} |
||||
|
||||
func (db *DB) Get(key []byte) ([]byte, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
key = db.encodeKVKey(key) |
||||
|
||||
return db.bucket.Get(key) |
||||
} |
||||
|
||||
func (db *DB) GetSet(key []byte, value []byte) ([]byte, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return nil, err |
||||
} else if err := checkValueSize(value); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
key = db.encodeKVKey(key) |
||||
|
||||
t := db.kvBatch |
||||
|
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
oldValue, err := db.bucket.Get(key) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
t.Put(key, value) |
||||
//todo, binlog
|
||||
|
||||
err = t.Commit() |
||||
|
||||
return oldValue, err |
||||
} |
||||
|
||||
func (db *DB) Incr(key []byte) (int64, error) { |
||||
return db.incr(key, 1) |
||||
} |
||||
|
||||
func (db *DB) IncrBy(key []byte, increment int64) (int64, error) { |
||||
return db.incr(key, increment) |
||||
} |
||||
|
||||
func (db *DB) MGet(keys ...[]byte) ([][]byte, error) { |
||||
values := make([][]byte, len(keys)) |
||||
|
||||
it := db.bucket.NewIterator() |
||||
defer it.Close() |
||||
|
||||
for i := range keys { |
||||
if err := checkKeySize(keys[i]); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
values[i] = it.Find(db.encodeKVKey(keys[i])) |
||||
} |
||||
|
||||
return values, nil |
||||
} |
||||
|
||||
func (db *DB) MSet(args ...KVPair) error { |
||||
if len(args) == 0 { |
||||
return nil |
||||
} |
||||
|
||||
t := db.kvBatch |
||||
|
||||
var err error |
||||
var key []byte |
||||
var value []byte |
||||
|
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
for i := 0; i < len(args); i++ { |
||||
if err := checkKeySize(args[i].Key); err != nil { |
||||
return err |
||||
} else if err := checkValueSize(args[i].Value); err != nil { |
||||
return err |
||||
} |
||||
|
||||
key = db.encodeKVKey(args[i].Key) |
||||
|
||||
value = args[i].Value |
||||
|
||||
t.Put(key, value) |
||||
|
||||
//todo binlog
|
||||
} |
||||
|
||||
err = t.Commit() |
||||
return err |
||||
} |
||||
|
||||
func (db *DB) Set(key []byte, value []byte) error { |
||||
if err := checkKeySize(key); err != nil { |
||||
return err |
||||
} else if err := checkValueSize(value); err != nil { |
||||
return err |
||||
} |
||||
|
||||
var err error |
||||
key = db.encodeKVKey(key) |
||||
|
||||
t := db.kvBatch |
||||
|
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
t.Put(key, value) |
||||
|
||||
err = t.Commit() |
||||
|
||||
return err |
||||
} |
||||
|
||||
func (db *DB) SetNX(key []byte, value []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} else if err := checkValueSize(value); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
var err error |
||||
key = db.encodeKVKey(key) |
||||
|
||||
var n int64 = 1 |
||||
|
||||
t := db.kvBatch |
||||
|
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
if v, err := db.bucket.Get(key); err != nil { |
||||
return 0, err |
||||
} else if v != nil { |
||||
n = 0 |
||||
} else { |
||||
t.Put(key, value) |
||||
|
||||
//todo binlog
|
||||
|
||||
err = t.Commit() |
||||
} |
||||
|
||||
return n, err |
||||
} |
||||
|
||||
func (db *DB) flush() (drop int64, err error) { |
||||
t := db.kvBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
return db.flushType(t, KVType) |
||||
} |
||||
|
||||
//if inclusive is true, scan range [key, inf) else (key, inf)
|
||||
func (db *DB) Scan(key []byte, count int, inclusive bool, match string) ([][]byte, error) { |
||||
return db.scan(KVType, key, count, inclusive, match) |
||||
} |
||||
|
||||
func (db *DB) Expire(key []byte, duration int64) (int64, error) { |
||||
if duration <= 0 { |
||||
return 0, errExpireValue |
||||
} |
||||
|
||||
return db.setExpireAt(key, time.Now().Unix()+duration) |
||||
} |
||||
|
||||
func (db *DB) ExpireAt(key []byte, when int64) (int64, error) { |
||||
if when <= time.Now().Unix() { |
||||
return 0, errExpireValue |
||||
} |
||||
|
||||
return db.setExpireAt(key, when) |
||||
} |
||||
|
||||
func (db *DB) TTL(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return -1, err |
||||
} |
||||
|
||||
return db.ttl(KVType, key) |
||||
} |
||||
|
||||
func (db *DB) Persist(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
t := db.kvBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
n, err := db.rmExpire(t, KVType, key) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
err = t.Commit() |
||||
return n, err |
||||
} |
||||
|
||||
func (db *DB) Lock() { |
||||
t := db.kvBatch |
||||
t.Lock() |
||||
} |
||||
|
||||
func (db *DB) Remove(key []byte) bool { |
||||
if len(key) == 0 { |
||||
return false |
||||
} |
||||
t := db.kvBatch |
||||
t.Delete(db.encodeKVKey(key)) |
||||
_, err := db.rmExpire(t, KVType, key) |
||||
if err != nil { |
||||
return false |
||||
} |
||||
return true |
||||
} |
||||
|
||||
func (db *DB) Commit() error { |
||||
t := db.kvBatch |
||||
return t.Commit() |
||||
} |
||||
|
||||
func (db *DB) Unlock() { |
||||
t := db.kvBatch |
||||
t.Unlock() |
||||
} |
||||
@ -0,0 +1,492 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
"errors" |
||||
"time" |
||||
|
||||
"github.com/lunny/nodb/store" |
||||
) |
||||
|
||||
const ( |
||||
listHeadSeq int32 = 1 |
||||
listTailSeq int32 = 2 |
||||
|
||||
listMinSeq int32 = 1000 |
||||
listMaxSeq int32 = 1<<31 - 1000 |
||||
listInitialSeq int32 = listMinSeq + (listMaxSeq-listMinSeq)/2 |
||||
) |
||||
|
||||
var errLMetaKey = errors.New("invalid lmeta key") |
||||
var errListKey = errors.New("invalid list key") |
||||
var errListSeq = errors.New("invalid list sequence, overflow") |
||||
|
||||
func (db *DB) lEncodeMetaKey(key []byte) []byte { |
||||
buf := make([]byte, len(key)+2) |
||||
buf[0] = db.index |
||||
buf[1] = LMetaType |
||||
|
||||
copy(buf[2:], key) |
||||
return buf |
||||
} |
||||
|
||||
func (db *DB) lDecodeMetaKey(ek []byte) ([]byte, error) { |
||||
if len(ek) < 2 || ek[0] != db.index || ek[1] != LMetaType { |
||||
return nil, errLMetaKey |
||||
} |
||||
|
||||
return ek[2:], nil |
||||
} |
||||
|
||||
func (db *DB) lEncodeListKey(key []byte, seq int32) []byte { |
||||
buf := make([]byte, len(key)+8) |
||||
|
||||
pos := 0 |
||||
buf[pos] = db.index |
||||
pos++ |
||||
buf[pos] = ListType |
||||
pos++ |
||||
|
||||
binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) |
||||
pos += 2 |
||||
|
||||
copy(buf[pos:], key) |
||||
pos += len(key) |
||||
|
||||
binary.BigEndian.PutUint32(buf[pos:], uint32(seq)) |
||||
|
||||
return buf |
||||
} |
||||
|
||||
func (db *DB) lDecodeListKey(ek []byte) (key []byte, seq int32, err error) { |
||||
if len(ek) < 8 || ek[0] != db.index || ek[1] != ListType { |
||||
err = errListKey |
||||
return |
||||
} |
||||
|
||||
keyLen := int(binary.BigEndian.Uint16(ek[2:])) |
||||
if keyLen+8 != len(ek) { |
||||
err = errListKey |
||||
return |
||||
} |
||||
|
||||
key = ek[4 : 4+keyLen] |
||||
seq = int32(binary.BigEndian.Uint32(ek[4+keyLen:])) |
||||
return |
||||
} |
||||
|
||||
func (db *DB) lpush(key []byte, whereSeq int32, args ...[]byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
var headSeq int32 |
||||
var tailSeq int32 |
||||
var size int32 |
||||
var err error |
||||
|
||||
t := db.listBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
metaKey := db.lEncodeMetaKey(key) |
||||
headSeq, tailSeq, size, err = db.lGetMeta(nil, metaKey) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
var pushCnt int = len(args) |
||||
if pushCnt == 0 { |
||||
return int64(size), nil |
||||
} |
||||
|
||||
var seq int32 = headSeq |
||||
var delta int32 = -1 |
||||
if whereSeq == listTailSeq { |
||||
seq = tailSeq |
||||
delta = 1 |
||||
} |
||||
|
||||
// append elements
|
||||
if size > 0 { |
||||
seq += delta |
||||
} |
||||
|
||||
for i := 0; i < pushCnt; i++ { |
||||
ek := db.lEncodeListKey(key, seq+int32(i)*delta) |
||||
t.Put(ek, args[i]) |
||||
} |
||||
|
||||
seq += int32(pushCnt-1) * delta |
||||
if seq <= listMinSeq || seq >= listMaxSeq { |
||||
return 0, errListSeq |
||||
} |
||||
|
||||
// set meta info
|
||||
if whereSeq == listHeadSeq { |
||||
headSeq = seq |
||||
} else { |
||||
tailSeq = seq |
||||
} |
||||
|
||||
db.lSetMeta(metaKey, headSeq, tailSeq) |
||||
|
||||
err = t.Commit() |
||||
return int64(size) + int64(pushCnt), err |
||||
} |
||||
|
||||
func (db *DB) lpop(key []byte, whereSeq int32) ([]byte, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
t := db.listBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
var headSeq int32 |
||||
var tailSeq int32 |
||||
var err error |
||||
|
||||
metaKey := db.lEncodeMetaKey(key) |
||||
headSeq, tailSeq, _, err = db.lGetMeta(nil, metaKey) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
var value []byte |
||||
|
||||
var seq int32 = headSeq |
||||
if whereSeq == listTailSeq { |
||||
seq = tailSeq |
||||
} |
||||
|
||||
itemKey := db.lEncodeListKey(key, seq) |
||||
value, err = db.bucket.Get(itemKey) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if whereSeq == listHeadSeq { |
||||
headSeq += 1 |
||||
} else { |
||||
tailSeq -= 1 |
||||
} |
||||
|
||||
t.Delete(itemKey) |
||||
size := db.lSetMeta(metaKey, headSeq, tailSeq) |
||||
if size == 0 { |
||||
db.rmExpire(t, HashType, key) |
||||
} |
||||
|
||||
err = t.Commit() |
||||
return value, err |
||||
} |
||||
|
||||
// ps : here just focus on deleting the list data,
|
||||
// any other likes expire is ignore.
|
||||
func (db *DB) lDelete(t *batch, key []byte) int64 { |
||||
mk := db.lEncodeMetaKey(key) |
||||
|
||||
var headSeq int32 |
||||
var tailSeq int32 |
||||
var err error |
||||
|
||||
it := db.bucket.NewIterator() |
||||
defer it.Close() |
||||
|
||||
headSeq, tailSeq, _, err = db.lGetMeta(it, mk) |
||||
if err != nil { |
||||
return 0 |
||||
} |
||||
|
||||
var num int64 = 0 |
||||
startKey := db.lEncodeListKey(key, headSeq) |
||||
stopKey := db.lEncodeListKey(key, tailSeq) |
||||
|
||||
rit := store.NewRangeIterator(it, &store.Range{startKey, stopKey, store.RangeClose}) |
||||
for ; rit.Valid(); rit.Next() { |
||||
t.Delete(rit.RawKey()) |
||||
num++ |
||||
} |
||||
|
||||
t.Delete(mk) |
||||
|
||||
return num |
||||
} |
||||
|
||||
func (db *DB) lGetMeta(it *store.Iterator, ek []byte) (headSeq int32, tailSeq int32, size int32, err error) { |
||||
var v []byte |
||||
if it != nil { |
||||
v = it.Find(ek) |
||||
} else { |
||||
v, err = db.bucket.Get(ek) |
||||
} |
||||
if err != nil { |
||||
return |
||||
} else if v == nil { |
||||
headSeq = listInitialSeq |
||||
tailSeq = listInitialSeq |
||||
size = 0 |
||||
return |
||||
} else { |
||||
headSeq = int32(binary.LittleEndian.Uint32(v[0:4])) |
||||
tailSeq = int32(binary.LittleEndian.Uint32(v[4:8])) |
||||
size = tailSeq - headSeq + 1 |
||||
} |
||||
return |
||||
} |
||||
|
||||
func (db *DB) lSetMeta(ek []byte, headSeq int32, tailSeq int32) int32 { |
||||
t := db.listBatch |
||||
|
||||
var size int32 = tailSeq - headSeq + 1 |
||||
if size < 0 { |
||||
// todo : log error + panic
|
||||
} else if size == 0 { |
||||
t.Delete(ek) |
||||
} else { |
||||
buf := make([]byte, 8) |
||||
|
||||
binary.LittleEndian.PutUint32(buf[0:4], uint32(headSeq)) |
||||
binary.LittleEndian.PutUint32(buf[4:8], uint32(tailSeq)) |
||||
|
||||
t.Put(ek, buf) |
||||
} |
||||
|
||||
return size |
||||
} |
||||
|
||||
func (db *DB) lExpireAt(key []byte, when int64) (int64, error) { |
||||
t := db.listBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
if llen, err := db.LLen(key); err != nil || llen == 0 { |
||||
return 0, err |
||||
} else { |
||||
db.expireAt(t, ListType, key, when) |
||||
if err := t.Commit(); err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
return 1, nil |
||||
} |
||||
|
||||
func (db *DB) LIndex(key []byte, index int32) ([]byte, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
var seq int32 |
||||
var headSeq int32 |
||||
var tailSeq int32 |
||||
var err error |
||||
|
||||
metaKey := db.lEncodeMetaKey(key) |
||||
|
||||
it := db.bucket.NewIterator() |
||||
defer it.Close() |
||||
|
||||
headSeq, tailSeq, _, err = db.lGetMeta(it, metaKey) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if index >= 0 { |
||||
seq = headSeq + index |
||||
} else { |
||||
seq = tailSeq + index + 1 |
||||
} |
||||
|
||||
sk := db.lEncodeListKey(key, seq) |
||||
v := it.Find(sk) |
||||
|
||||
return v, nil |
||||
} |
||||
|
||||
func (db *DB) LLen(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
ek := db.lEncodeMetaKey(key) |
||||
_, _, size, err := db.lGetMeta(nil, ek) |
||||
return int64(size), err |
||||
} |
||||
|
||||
func (db *DB) LPop(key []byte) ([]byte, error) { |
||||
return db.lpop(key, listHeadSeq) |
||||
} |
||||
|
||||
func (db *DB) LPush(key []byte, arg1 []byte, args ...[]byte) (int64, error) { |
||||
var argss = [][]byte{arg1} |
||||
argss = append(argss, args...) |
||||
return db.lpush(key, listHeadSeq, argss...) |
||||
} |
||||
|
||||
func (db *DB) LRange(key []byte, start int32, stop int32) ([][]byte, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
var headSeq int32 |
||||
var llen int32 |
||||
var err error |
||||
|
||||
metaKey := db.lEncodeMetaKey(key) |
||||
|
||||
it := db.bucket.NewIterator() |
||||
defer it.Close() |
||||
|
||||
if headSeq, _, llen, err = db.lGetMeta(it, metaKey); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if start < 0 { |
||||
start = llen + start |
||||
} |
||||
if stop < 0 { |
||||
stop = llen + stop |
||||
} |
||||
if start < 0 { |
||||
start = 0 |
||||
} |
||||
|
||||
if start > stop || start >= llen { |
||||
return [][]byte{}, nil |
||||
} |
||||
|
||||
if stop >= llen { |
||||
stop = llen - 1 |
||||
} |
||||
|
||||
limit := (stop - start) + 1 |
||||
headSeq += start |
||||
|
||||
v := make([][]byte, 0, limit) |
||||
|
||||
startKey := db.lEncodeListKey(key, headSeq) |
||||
rit := store.NewRangeLimitIterator(it, |
||||
&store.Range{ |
||||
Min: startKey, |
||||
Max: nil, |
||||
Type: store.RangeClose}, |
||||
&store.Limit{ |
||||
Offset: 0, |
||||
Count: int(limit)}) |
||||
|
||||
for ; rit.Valid(); rit.Next() { |
||||
v = append(v, rit.Value()) |
||||
} |
||||
|
||||
return v, nil |
||||
} |
||||
|
||||
func (db *DB) RPop(key []byte) ([]byte, error) { |
||||
return db.lpop(key, listTailSeq) |
||||
} |
||||
|
||||
func (db *DB) RPush(key []byte, arg1 []byte, args ...[]byte) (int64, error) { |
||||
var argss = [][]byte{arg1} |
||||
argss = append(argss, args...) |
||||
return db.lpush(key, listTailSeq, argss...) |
||||
} |
||||
|
||||
func (db *DB) LClear(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
t := db.listBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
num := db.lDelete(t, key) |
||||
db.rmExpire(t, ListType, key) |
||||
|
||||
err := t.Commit() |
||||
return num, err |
||||
} |
||||
|
||||
func (db *DB) LMclear(keys ...[]byte) (int64, error) { |
||||
t := db.listBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
for _, key := range keys { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
db.lDelete(t, key) |
||||
db.rmExpire(t, ListType, key) |
||||
|
||||
} |
||||
|
||||
err := t.Commit() |
||||
return int64(len(keys)), err |
||||
} |
||||
|
||||
func (db *DB) lFlush() (drop int64, err error) { |
||||
t := db.listBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
return db.flushType(t, ListType) |
||||
} |
||||
|
||||
func (db *DB) LExpire(key []byte, duration int64) (int64, error) { |
||||
if duration <= 0 { |
||||
return 0, errExpireValue |
||||
} |
||||
|
||||
return db.lExpireAt(key, time.Now().Unix()+duration) |
||||
} |
||||
|
||||
func (db *DB) LExpireAt(key []byte, when int64) (int64, error) { |
||||
if when <= time.Now().Unix() { |
||||
return 0, errExpireValue |
||||
} |
||||
|
||||
return db.lExpireAt(key, when) |
||||
} |
||||
|
||||
func (db *DB) LTTL(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return -1, err |
||||
} |
||||
|
||||
return db.ttl(ListType, key) |
||||
} |
||||
|
||||
func (db *DB) LPersist(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
t := db.listBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
n, err := db.rmExpire(t, ListType, key) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
err = t.Commit() |
||||
return n, err |
||||
} |
||||
|
||||
func (db *DB) LScan(key []byte, count int, inclusive bool, match string) ([][]byte, error) { |
||||
return db.scan(LMetaType, key, count, inclusive, match) |
||||
} |
||||
|
||||
func (db *DB) lEncodeMinKey() []byte { |
||||
return db.lEncodeMetaKey(nil) |
||||
} |
||||
|
||||
func (db *DB) lEncodeMaxKey() []byte { |
||||
ek := db.lEncodeMetaKey(nil) |
||||
ek[len(ek)-1] = LMetaType + 1 |
||||
return ek |
||||
} |
||||
@ -0,0 +1,601 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
"errors" |
||||
"time" |
||||
|
||||
"github.com/lunny/nodb/store" |
||||
) |
||||
|
||||
var errSetKey = errors.New("invalid set key") |
||||
var errSSizeKey = errors.New("invalid ssize key") |
||||
|
||||
const ( |
||||
setStartSep byte = ':' |
||||
setStopSep byte = setStartSep + 1 |
||||
UnionType byte = 51 |
||||
DiffType byte = 52 |
||||
InterType byte = 53 |
||||
) |
||||
|
||||
func checkSetKMSize(key []byte, member []byte) error { |
||||
if len(key) > MaxKeySize || len(key) == 0 { |
||||
return errKeySize |
||||
} else if len(member) > MaxSetMemberSize || len(member) == 0 { |
||||
return errSetMemberSize |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (db *DB) sEncodeSizeKey(key []byte) []byte { |
||||
buf := make([]byte, len(key)+2) |
||||
|
||||
buf[0] = db.index |
||||
buf[1] = SSizeType |
||||
|
||||
copy(buf[2:], key) |
||||
return buf |
||||
} |
||||
|
||||
func (db *DB) sDecodeSizeKey(ek []byte) ([]byte, error) { |
||||
if len(ek) < 2 || ek[0] != db.index || ek[1] != SSizeType { |
||||
return nil, errSSizeKey |
||||
} |
||||
|
||||
return ek[2:], nil |
||||
} |
||||
|
||||
func (db *DB) sEncodeSetKey(key []byte, member []byte) []byte { |
||||
buf := make([]byte, len(key)+len(member)+1+1+2+1) |
||||
|
||||
pos := 0 |
||||
buf[pos] = db.index |
||||
pos++ |
||||
buf[pos] = SetType |
||||
pos++ |
||||
|
||||
binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) |
||||
pos += 2 |
||||
|
||||
copy(buf[pos:], key) |
||||
pos += len(key) |
||||
|
||||
buf[pos] = setStartSep |
||||
pos++ |
||||
copy(buf[pos:], member) |
||||
|
||||
return buf |
||||
} |
||||
|
||||
func (db *DB) sDecodeSetKey(ek []byte) ([]byte, []byte, error) { |
||||
if len(ek) < 5 || ek[0] != db.index || ek[1] != SetType { |
||||
return nil, nil, errSetKey |
||||
} |
||||
|
||||
pos := 2 |
||||
keyLen := int(binary.BigEndian.Uint16(ek[pos:])) |
||||
pos += 2 |
||||
|
||||
if keyLen+5 > len(ek) { |
||||
return nil, nil, errSetKey |
||||
} |
||||
|
||||
key := ek[pos : pos+keyLen] |
||||
pos += keyLen |
||||
|
||||
if ek[pos] != hashStartSep { |
||||
return nil, nil, errSetKey |
||||
} |
||||
|
||||
pos++ |
||||
member := ek[pos:] |
||||
return key, member, nil |
||||
} |
||||
|
||||
func (db *DB) sEncodeStartKey(key []byte) []byte { |
||||
return db.sEncodeSetKey(key, nil) |
||||
} |
||||
|
||||
func (db *DB) sEncodeStopKey(key []byte) []byte { |
||||
k := db.sEncodeSetKey(key, nil) |
||||
|
||||
k[len(k)-1] = setStopSep |
||||
|
||||
return k |
||||
} |
||||
|
||||
func (db *DB) sFlush() (drop int64, err error) { |
||||
|
||||
t := db.setBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
return db.flushType(t, SetType) |
||||
} |
||||
|
||||
func (db *DB) sDelete(t *batch, key []byte) int64 { |
||||
sk := db.sEncodeSizeKey(key) |
||||
start := db.sEncodeStartKey(key) |
||||
stop := db.sEncodeStopKey(key) |
||||
|
||||
var num int64 = 0 |
||||
it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) |
||||
for ; it.Valid(); it.Next() { |
||||
t.Delete(it.RawKey()) |
||||
num++ |
||||
} |
||||
|
||||
it.Close() |
||||
|
||||
t.Delete(sk) |
||||
return num |
||||
} |
||||
|
||||
func (db *DB) sIncrSize(key []byte, delta int64) (int64, error) { |
||||
t := db.setBatch |
||||
sk := db.sEncodeSizeKey(key) |
||||
|
||||
var err error |
||||
var size int64 = 0 |
||||
if size, err = Int64(db.bucket.Get(sk)); err != nil { |
||||
return 0, err |
||||
} else { |
||||
size += delta |
||||
if size <= 0 { |
||||
size = 0 |
||||
t.Delete(sk) |
||||
db.rmExpire(t, SetType, key) |
||||
} else { |
||||
t.Put(sk, PutInt64(size)) |
||||
} |
||||
} |
||||
|
||||
return size, nil |
||||
} |
||||
|
||||
func (db *DB) sExpireAt(key []byte, when int64) (int64, error) { |
||||
t := db.setBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
if scnt, err := db.SCard(key); err != nil || scnt == 0 { |
||||
return 0, err |
||||
} else { |
||||
db.expireAt(t, SetType, key, when) |
||||
if err := t.Commit(); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
} |
||||
|
||||
return 1, nil |
||||
} |
||||
|
||||
func (db *DB) sSetItem(key []byte, member []byte) (int64, error) { |
||||
t := db.setBatch |
||||
ek := db.sEncodeSetKey(key, member) |
||||
|
||||
var n int64 = 1 |
||||
if v, _ := db.bucket.Get(ek); v != nil { |
||||
n = 0 |
||||
} else { |
||||
if _, err := db.sIncrSize(key, 1); err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
|
||||
t.Put(ek, nil) |
||||
return n, nil |
||||
} |
||||
|
||||
func (db *DB) SAdd(key []byte, args ...[]byte) (int64, error) { |
||||
t := db.setBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
var err error |
||||
var ek []byte |
||||
var num int64 = 0 |
||||
for i := 0; i < len(args); i++ { |
||||
if err := checkSetKMSize(key, args[i]); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
ek = db.sEncodeSetKey(key, args[i]) |
||||
|
||||
if v, err := db.bucket.Get(ek); err != nil { |
||||
return 0, err |
||||
} else if v == nil { |
||||
num++ |
||||
} |
||||
|
||||
t.Put(ek, nil) |
||||
} |
||||
|
||||
if _, err = db.sIncrSize(key, num); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
err = t.Commit() |
||||
return num, err |
||||
|
||||
} |
||||
|
||||
func (db *DB) SCard(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
sk := db.sEncodeSizeKey(key) |
||||
|
||||
return Int64(db.bucket.Get(sk)) |
||||
} |
||||
|
||||
func (db *DB) sDiffGeneric(keys ...[]byte) ([][]byte, error) { |
||||
destMap := make(map[string]bool) |
||||
|
||||
members, err := db.SMembers(keys[0]) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
for _, m := range members { |
||||
destMap[String(m)] = true |
||||
} |
||||
|
||||
for _, k := range keys[1:] { |
||||
members, err := db.SMembers(k) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
for _, m := range members { |
||||
if _, ok := destMap[String(m)]; !ok { |
||||
continue |
||||
} else if ok { |
||||
delete(destMap, String(m)) |
||||
} |
||||
} |
||||
// O - A = O, O is zero set.
|
||||
if len(destMap) == 0 { |
||||
return nil, nil |
||||
} |
||||
} |
||||
|
||||
slice := make([][]byte, len(destMap)) |
||||
idx := 0 |
||||
for k, v := range destMap { |
||||
if !v { |
||||
continue |
||||
} |
||||
slice[idx] = []byte(k) |
||||
idx++ |
||||
} |
||||
|
||||
return slice, nil |
||||
} |
||||
|
||||
func (db *DB) SDiff(keys ...[]byte) ([][]byte, error) { |
||||
v, err := db.sDiffGeneric(keys...) |
||||
return v, err |
||||
} |
||||
|
||||
func (db *DB) SDiffStore(dstKey []byte, keys ...[]byte) (int64, error) { |
||||
n, err := db.sStoreGeneric(dstKey, DiffType, keys...) |
||||
return n, err |
||||
} |
||||
|
||||
func (db *DB) sInterGeneric(keys ...[]byte) ([][]byte, error) { |
||||
destMap := make(map[string]bool) |
||||
|
||||
members, err := db.SMembers(keys[0]) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
for _, m := range members { |
||||
destMap[String(m)] = true |
||||
} |
||||
|
||||
for _, key := range keys[1:] { |
||||
if err := checkKeySize(key); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
members, err := db.SMembers(key) |
||||
if err != nil { |
||||
return nil, err |
||||
} else if len(members) == 0 { |
||||
return nil, err |
||||
} |
||||
|
||||
tempMap := make(map[string]bool) |
||||
for _, member := range members { |
||||
if err := checkKeySize(member); err != nil { |
||||
return nil, err |
||||
} |
||||
if _, ok := destMap[String(member)]; ok { |
||||
tempMap[String(member)] = true //mark this item as selected
|
||||
} |
||||
} |
||||
destMap = tempMap //reduce the size of the result set
|
||||
if len(destMap) == 0 { |
||||
return nil, nil |
||||
} |
||||
} |
||||
|
||||
slice := make([][]byte, len(destMap)) |
||||
idx := 0 |
||||
for k, v := range destMap { |
||||
if !v { |
||||
continue |
||||
} |
||||
|
||||
slice[idx] = []byte(k) |
||||
idx++ |
||||
} |
||||
|
||||
return slice, nil |
||||
|
||||
} |
||||
|
||||
func (db *DB) SInter(keys ...[]byte) ([][]byte, error) { |
||||
v, err := db.sInterGeneric(keys...) |
||||
return v, err |
||||
|
||||
} |
||||
|
||||
func (db *DB) SInterStore(dstKey []byte, keys ...[]byte) (int64, error) { |
||||
n, err := db.sStoreGeneric(dstKey, InterType, keys...) |
||||
return n, err |
||||
} |
||||
|
||||
func (db *DB) SIsMember(key []byte, member []byte) (int64, error) { |
||||
ek := db.sEncodeSetKey(key, member) |
||||
|
||||
var n int64 = 1 |
||||
if v, err := db.bucket.Get(ek); err != nil { |
||||
return 0, err |
||||
} else if v == nil { |
||||
n = 0 |
||||
} |
||||
return n, nil |
||||
} |
||||
|
||||
func (db *DB) SMembers(key []byte) ([][]byte, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
start := db.sEncodeStartKey(key) |
||||
stop := db.sEncodeStopKey(key) |
||||
|
||||
v := make([][]byte, 0, 16) |
||||
|
||||
it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) |
||||
for ; it.Valid(); it.Next() { |
||||
_, m, err := db.sDecodeSetKey(it.Key()) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
v = append(v, m) |
||||
} |
||||
|
||||
it.Close() |
||||
|
||||
return v, nil |
||||
} |
||||
|
||||
func (db *DB) SRem(key []byte, args ...[]byte) (int64, error) { |
||||
t := db.setBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
var ek []byte |
||||
var v []byte |
||||
var err error |
||||
|
||||
it := db.bucket.NewIterator() |
||||
defer it.Close() |
||||
|
||||
var num int64 = 0 |
||||
for i := 0; i < len(args); i++ { |
||||
if err := checkSetKMSize(key, args[i]); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
ek = db.sEncodeSetKey(key, args[i]) |
||||
|
||||
v = it.RawFind(ek) |
||||
if v == nil { |
||||
continue |
||||
} else { |
||||
num++ |
||||
t.Delete(ek) |
||||
} |
||||
} |
||||
|
||||
if _, err = db.sIncrSize(key, -num); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
err = t.Commit() |
||||
return num, err |
||||
|
||||
} |
||||
|
||||
func (db *DB) sUnionGeneric(keys ...[]byte) ([][]byte, error) { |
||||
dstMap := make(map[string]bool) |
||||
|
||||
for _, key := range keys { |
||||
if err := checkKeySize(key); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
members, err := db.SMembers(key) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
for _, member := range members { |
||||
dstMap[String(member)] = true |
||||
} |
||||
} |
||||
|
||||
slice := make([][]byte, len(dstMap)) |
||||
idx := 0 |
||||
for k, v := range dstMap { |
||||
if !v { |
||||
continue |
||||
} |
||||
slice[idx] = []byte(k) |
||||
idx++ |
||||
} |
||||
|
||||
return slice, nil |
||||
} |
||||
|
||||
func (db *DB) SUnion(keys ...[]byte) ([][]byte, error) { |
||||
v, err := db.sUnionGeneric(keys...) |
||||
return v, err |
||||
} |
||||
|
||||
func (db *DB) SUnionStore(dstKey []byte, keys ...[]byte) (int64, error) { |
||||
n, err := db.sStoreGeneric(dstKey, UnionType, keys...) |
||||
return n, err |
||||
} |
||||
|
||||
func (db *DB) sStoreGeneric(dstKey []byte, optType byte, keys ...[]byte) (int64, error) { |
||||
if err := checkKeySize(dstKey); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
t := db.setBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
db.sDelete(t, dstKey) |
||||
|
||||
var err error |
||||
var ek []byte |
||||
var v [][]byte |
||||
|
||||
switch optType { |
||||
case UnionType: |
||||
v, err = db.sUnionGeneric(keys...) |
||||
case DiffType: |
||||
v, err = db.sDiffGeneric(keys...) |
||||
case InterType: |
||||
v, err = db.sInterGeneric(keys...) |
||||
} |
||||
|
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
for _, m := range v { |
||||
if err := checkSetKMSize(dstKey, m); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
ek = db.sEncodeSetKey(dstKey, m) |
||||
|
||||
if _, err := db.bucket.Get(ek); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
t.Put(ek, nil) |
||||
} |
||||
|
||||
var num = int64(len(v)) |
||||
sk := db.sEncodeSizeKey(dstKey) |
||||
t.Put(sk, PutInt64(num)) |
||||
|
||||
if err = t.Commit(); err != nil { |
||||
return 0, err |
||||
} |
||||
return num, nil |
||||
} |
||||
|
||||
func (db *DB) SClear(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
t := db.setBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
num := db.sDelete(t, key) |
||||
db.rmExpire(t, SetType, key) |
||||
|
||||
err := t.Commit() |
||||
return num, err |
||||
} |
||||
|
||||
func (db *DB) SMclear(keys ...[]byte) (int64, error) { |
||||
t := db.setBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
for _, key := range keys { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
db.sDelete(t, key) |
||||
db.rmExpire(t, SetType, key) |
||||
} |
||||
|
||||
err := t.Commit() |
||||
return int64(len(keys)), err |
||||
} |
||||
|
||||
func (db *DB) SExpire(key []byte, duration int64) (int64, error) { |
||||
if duration <= 0 { |
||||
return 0, errExpireValue |
||||
} |
||||
|
||||
return db.sExpireAt(key, time.Now().Unix()+duration) |
||||
|
||||
} |
||||
|
||||
func (db *DB) SExpireAt(key []byte, when int64) (int64, error) { |
||||
if when <= time.Now().Unix() { |
||||
return 0, errExpireValue |
||||
} |
||||
|
||||
return db.sExpireAt(key, when) |
||||
|
||||
} |
||||
|
||||
func (db *DB) STTL(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return -1, err |
||||
} |
||||
|
||||
return db.ttl(SetType, key) |
||||
} |
||||
|
||||
func (db *DB) SPersist(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
t := db.setBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
n, err := db.rmExpire(t, SetType, key) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
err = t.Commit() |
||||
return n, err |
||||
} |
||||
|
||||
func (db *DB) SScan(key []byte, count int, inclusive bool, match string) ([][]byte, error) { |
||||
return db.scan(SSizeType, key, count, inclusive, match) |
||||
} |
||||
@ -0,0 +1,195 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
"errors" |
||||
"time" |
||||
|
||||
"github.com/lunny/nodb/store" |
||||
) |
||||
|
||||
var ( |
||||
errExpMetaKey = errors.New("invalid expire meta key") |
||||
errExpTimeKey = errors.New("invalid expire time key") |
||||
) |
||||
|
||||
type retireCallback func(*batch, []byte) int64 |
||||
|
||||
type elimination struct { |
||||
db *DB |
||||
exp2Tx []*batch |
||||
exp2Retire []retireCallback |
||||
} |
||||
|
||||
var errExpType = errors.New("invalid expire type") |
||||
|
||||
func (db *DB) expEncodeTimeKey(dataType byte, key []byte, when int64) []byte { |
||||
buf := make([]byte, len(key)+11) |
||||
|
||||
buf[0] = db.index |
||||
buf[1] = ExpTimeType |
||||
buf[2] = dataType |
||||
pos := 3 |
||||
|
||||
binary.BigEndian.PutUint64(buf[pos:], uint64(when)) |
||||
pos += 8 |
||||
|
||||
copy(buf[pos:], key) |
||||
|
||||
return buf |
||||
} |
||||
|
||||
func (db *DB) expEncodeMetaKey(dataType byte, key []byte) []byte { |
||||
buf := make([]byte, len(key)+3) |
||||
|
||||
buf[0] = db.index |
||||
buf[1] = ExpMetaType |
||||
buf[2] = dataType |
||||
pos := 3 |
||||
|
||||
copy(buf[pos:], key) |
||||
|
||||
return buf |
||||
} |
||||
|
||||
func (db *DB) expDecodeMetaKey(mk []byte) (byte, []byte, error) { |
||||
if len(mk) <= 3 || mk[0] != db.index || mk[1] != ExpMetaType { |
||||
return 0, nil, errExpMetaKey |
||||
} |
||||
|
||||
return mk[2], mk[3:], nil |
||||
} |
||||
|
||||
func (db *DB) expDecodeTimeKey(tk []byte) (byte, []byte, int64, error) { |
||||
if len(tk) < 11 || tk[0] != db.index || tk[1] != ExpTimeType { |
||||
return 0, nil, 0, errExpTimeKey |
||||
} |
||||
|
||||
return tk[2], tk[11:], int64(binary.BigEndian.Uint64(tk[3:])), nil |
||||
} |
||||
|
||||
func (db *DB) expire(t *batch, dataType byte, key []byte, duration int64) { |
||||
db.expireAt(t, dataType, key, time.Now().Unix()+duration) |
||||
} |
||||
|
||||
func (db *DB) expireAt(t *batch, dataType byte, key []byte, when int64) { |
||||
mk := db.expEncodeMetaKey(dataType, key) |
||||
tk := db.expEncodeTimeKey(dataType, key, when) |
||||
|
||||
t.Put(tk, mk) |
||||
t.Put(mk, PutInt64(when)) |
||||
} |
||||
|
||||
func (db *DB) ttl(dataType byte, key []byte) (t int64, err error) { |
||||
mk := db.expEncodeMetaKey(dataType, key) |
||||
|
||||
if t, err = Int64(db.bucket.Get(mk)); err != nil || t == 0 { |
||||
t = -1 |
||||
} else { |
||||
t -= time.Now().Unix() |
||||
if t <= 0 { |
||||
t = -1 |
||||
} |
||||
// if t == -1 : to remove ????
|
||||
} |
||||
|
||||
return t, err |
||||
} |
||||
|
||||
func (db *DB) rmExpire(t *batch, dataType byte, key []byte) (int64, error) { |
||||
mk := db.expEncodeMetaKey(dataType, key) |
||||
if v, err := db.bucket.Get(mk); err != nil { |
||||
return 0, err |
||||
} else if v == nil { |
||||
return 0, nil |
||||
} else if when, err2 := Int64(v, nil); err2 != nil { |
||||
return 0, err2 |
||||
} else { |
||||
tk := db.expEncodeTimeKey(dataType, key, when) |
||||
t.Delete(mk) |
||||
t.Delete(tk) |
||||
return 1, nil |
||||
} |
||||
} |
||||
|
||||
func (db *DB) expFlush(t *batch, dataType byte) (err error) { |
||||
minKey := make([]byte, 3) |
||||
minKey[0] = db.index |
||||
minKey[1] = ExpTimeType |
||||
minKey[2] = dataType |
||||
|
||||
maxKey := make([]byte, 3) |
||||
maxKey[0] = db.index |
||||
maxKey[1] = ExpMetaType |
||||
maxKey[2] = dataType + 1 |
||||
|
||||
_, err = db.flushRegion(t, minKey, maxKey) |
||||
err = t.Commit() |
||||
return |
||||
} |
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
//
|
||||
//////////////////////////////////////////////////////////
|
||||
|
||||
func newEliminator(db *DB) *elimination { |
||||
eli := new(elimination) |
||||
eli.db = db |
||||
eli.exp2Tx = make([]*batch, maxDataType) |
||||
eli.exp2Retire = make([]retireCallback, maxDataType) |
||||
return eli |
||||
} |
||||
|
||||
func (eli *elimination) regRetireContext(dataType byte, t *batch, onRetire retireCallback) { |
||||
|
||||
// todo .. need to ensure exist - mapExpMetaType[expType]
|
||||
|
||||
eli.exp2Tx[dataType] = t |
||||
eli.exp2Retire[dataType] = onRetire |
||||
} |
||||
|
||||
// call by outside ... (from *db to another *db)
|
||||
func (eli *elimination) active() { |
||||
now := time.Now().Unix() |
||||
db := eli.db |
||||
dbGet := db.bucket.Get |
||||
|
||||
minKey := db.expEncodeTimeKey(NoneType, nil, 0) |
||||
maxKey := db.expEncodeTimeKey(maxDataType, nil, now) |
||||
|
||||
it := db.bucket.RangeLimitIterator(minKey, maxKey, store.RangeROpen, 0, -1) |
||||
for ; it.Valid(); it.Next() { |
||||
tk := it.RawKey() |
||||
mk := it.RawValue() |
||||
|
||||
dt, k, _, err := db.expDecodeTimeKey(tk) |
||||
if err != nil { |
||||
continue |
||||
} |
||||
|
||||
t := eli.exp2Tx[dt] |
||||
onRetire := eli.exp2Retire[dt] |
||||
if tk == nil || onRetire == nil { |
||||
continue |
||||
} |
||||
|
||||
t.Lock() |
||||
|
||||
if exp, err := Int64(dbGet(mk)); err == nil { |
||||
// check expire again
|
||||
if exp <= now { |
||||
onRetire(t, k) |
||||
t.Delete(tk) |
||||
t.Delete(mk) |
||||
|
||||
t.Commit() |
||||
} |
||||
|
||||
} |
||||
|
||||
t.Unlock() |
||||
} |
||||
it.Close() |
||||
|
||||
return |
||||
} |
||||
@ -0,0 +1,943 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/binary" |
||||
"errors" |
||||
"time" |
||||
|
||||
"github.com/lunny/nodb/store" |
||||
) |
||||
|
||||
const ( |
||||
MinScore int64 = -1<<63 + 1 |
||||
MaxScore int64 = 1<<63 - 1 |
||||
InvalidScore int64 = -1 << 63 |
||||
|
||||
AggregateSum byte = 0 |
||||
AggregateMin byte = 1 |
||||
AggregateMax byte = 2 |
||||
) |
||||
|
||||
type ScorePair struct { |
||||
Score int64 |
||||
Member []byte |
||||
} |
||||
|
||||
var errZSizeKey = errors.New("invalid zsize key") |
||||
var errZSetKey = errors.New("invalid zset key") |
||||
var errZScoreKey = errors.New("invalid zscore key") |
||||
var errScoreOverflow = errors.New("zset score overflow") |
||||
var errInvalidAggregate = errors.New("invalid aggregate") |
||||
var errInvalidWeightNum = errors.New("invalid weight number") |
||||
var errInvalidSrcKeyNum = errors.New("invalid src key number") |
||||
|
||||
const ( |
||||
zsetNScoreSep byte = '<' |
||||
zsetPScoreSep byte = zsetNScoreSep + 1 |
||||
zsetStopScoreSep byte = zsetPScoreSep + 1 |
||||
|
||||
zsetStartMemSep byte = ':' |
||||
zsetStopMemSep byte = zsetStartMemSep + 1 |
||||
) |
||||
|
||||
func checkZSetKMSize(key []byte, member []byte) error { |
||||
if len(key) > MaxKeySize || len(key) == 0 { |
||||
return errKeySize |
||||
} else if len(member) > MaxZSetMemberSize || len(member) == 0 { |
||||
return errZSetMemberSize |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (db *DB) zEncodeSizeKey(key []byte) []byte { |
||||
buf := make([]byte, len(key)+2) |
||||
buf[0] = db.index |
||||
buf[1] = ZSizeType |
||||
|
||||
copy(buf[2:], key) |
||||
return buf |
||||
} |
||||
|
||||
func (db *DB) zDecodeSizeKey(ek []byte) ([]byte, error) { |
||||
if len(ek) < 2 || ek[0] != db.index || ek[1] != ZSizeType { |
||||
return nil, errZSizeKey |
||||
} |
||||
|
||||
return ek[2:], nil |
||||
} |
||||
|
||||
func (db *DB) zEncodeSetKey(key []byte, member []byte) []byte { |
||||
buf := make([]byte, len(key)+len(member)+5) |
||||
|
||||
pos := 0 |
||||
buf[pos] = db.index |
||||
pos++ |
||||
|
||||
buf[pos] = ZSetType |
||||
pos++ |
||||
|
||||
binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) |
||||
pos += 2 |
||||
|
||||
copy(buf[pos:], key) |
||||
pos += len(key) |
||||
|
||||
buf[pos] = zsetStartMemSep |
||||
pos++ |
||||
|
||||
copy(buf[pos:], member) |
||||
|
||||
return buf |
||||
} |
||||
|
||||
func (db *DB) zDecodeSetKey(ek []byte) ([]byte, []byte, error) { |
||||
if len(ek) < 5 || ek[0] != db.index || ek[1] != ZSetType { |
||||
return nil, nil, errZSetKey |
||||
} |
||||
|
||||
keyLen := int(binary.BigEndian.Uint16(ek[2:])) |
||||
if keyLen+5 > len(ek) { |
||||
return nil, nil, errZSetKey |
||||
} |
||||
|
||||
key := ek[4 : 4+keyLen] |
||||
|
||||
if ek[4+keyLen] != zsetStartMemSep { |
||||
return nil, nil, errZSetKey |
||||
} |
||||
|
||||
member := ek[5+keyLen:] |
||||
return key, member, nil |
||||
} |
||||
|
||||
func (db *DB) zEncodeStartSetKey(key []byte) []byte { |
||||
k := db.zEncodeSetKey(key, nil) |
||||
return k |
||||
} |
||||
|
||||
func (db *DB) zEncodeStopSetKey(key []byte) []byte { |
||||
k := db.zEncodeSetKey(key, nil) |
||||
k[len(k)-1] = zsetStartMemSep + 1 |
||||
return k |
||||
} |
||||
|
||||
func (db *DB) zEncodeScoreKey(key []byte, member []byte, score int64) []byte { |
||||
buf := make([]byte, len(key)+len(member)+14) |
||||
|
||||
pos := 0 |
||||
buf[pos] = db.index |
||||
pos++ |
||||
|
||||
buf[pos] = ZScoreType |
||||
pos++ |
||||
|
||||
binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) |
||||
pos += 2 |
||||
|
||||
copy(buf[pos:], key) |
||||
pos += len(key) |
||||
|
||||
if score < 0 { |
||||
buf[pos] = zsetNScoreSep |
||||
} else { |
||||
buf[pos] = zsetPScoreSep |
||||
} |
||||
|
||||
pos++ |
||||
binary.BigEndian.PutUint64(buf[pos:], uint64(score)) |
||||
pos += 8 |
||||
|
||||
buf[pos] = zsetStartMemSep |
||||
pos++ |
||||
|
||||
copy(buf[pos:], member) |
||||
return buf |
||||
} |
||||
|
||||
func (db *DB) zEncodeStartScoreKey(key []byte, score int64) []byte { |
||||
return db.zEncodeScoreKey(key, nil, score) |
||||
} |
||||
|
||||
func (db *DB) zEncodeStopScoreKey(key []byte, score int64) []byte { |
||||
k := db.zEncodeScoreKey(key, nil, score) |
||||
k[len(k)-1] = zsetStopMemSep |
||||
return k |
||||
} |
||||
|
||||
func (db *DB) zDecodeScoreKey(ek []byte) (key []byte, member []byte, score int64, err error) { |
||||
if len(ek) < 14 || ek[0] != db.index || ek[1] != ZScoreType { |
||||
err = errZScoreKey |
||||
return |
||||
} |
||||
|
||||
keyLen := int(binary.BigEndian.Uint16(ek[2:])) |
||||
if keyLen+14 > len(ek) { |
||||
err = errZScoreKey |
||||
return |
||||
} |
||||
|
||||
key = ek[4 : 4+keyLen] |
||||
pos := 4 + keyLen |
||||
|
||||
if (ek[pos] != zsetNScoreSep) && (ek[pos] != zsetPScoreSep) { |
||||
err = errZScoreKey |
||||
return |
||||
} |
||||
pos++ |
||||
|
||||
score = int64(binary.BigEndian.Uint64(ek[pos:])) |
||||
pos += 8 |
||||
|
||||
if ek[pos] != zsetStartMemSep { |
||||
err = errZScoreKey |
||||
return |
||||
} |
||||
|
||||
pos++ |
||||
|
||||
member = ek[pos:] |
||||
return |
||||
} |
||||
|
||||
func (db *DB) zSetItem(t *batch, key []byte, score int64, member []byte) (int64, error) { |
||||
if score <= MinScore || score >= MaxScore { |
||||
return 0, errScoreOverflow |
||||
} |
||||
|
||||
var exists int64 = 0 |
||||
ek := db.zEncodeSetKey(key, member) |
||||
|
||||
if v, err := db.bucket.Get(ek); err != nil { |
||||
return 0, err |
||||
} else if v != nil { |
||||
exists = 1 |
||||
|
||||
if s, err := Int64(v, err); err != nil { |
||||
return 0, err |
||||
} else { |
||||
sk := db.zEncodeScoreKey(key, member, s) |
||||
t.Delete(sk) |
||||
} |
||||
} |
||||
|
||||
t.Put(ek, PutInt64(score)) |
||||
|
||||
sk := db.zEncodeScoreKey(key, member, score) |
||||
t.Put(sk, []byte{}) |
||||
|
||||
return exists, nil |
||||
} |
||||
|
||||
func (db *DB) zDelItem(t *batch, key []byte, member []byte, skipDelScore bool) (int64, error) { |
||||
ek := db.zEncodeSetKey(key, member) |
||||
if v, err := db.bucket.Get(ek); err != nil { |
||||
return 0, err |
||||
} else if v == nil { |
||||
//not exists
|
||||
return 0, nil |
||||
} else { |
||||
//exists
|
||||
if !skipDelScore { |
||||
//we must del score
|
||||
if s, err := Int64(v, err); err != nil { |
||||
return 0, err |
||||
} else { |
||||
sk := db.zEncodeScoreKey(key, member, s) |
||||
t.Delete(sk) |
||||
} |
||||
} |
||||
} |
||||
|
||||
t.Delete(ek) |
||||
|
||||
return 1, nil |
||||
} |
||||
|
||||
func (db *DB) zDelete(t *batch, key []byte) int64 { |
||||
delMembCnt, _ := db.zRemRange(t, key, MinScore, MaxScore, 0, -1) |
||||
// todo : log err
|
||||
return delMembCnt |
||||
} |
||||
|
||||
func (db *DB) zExpireAt(key []byte, when int64) (int64, error) { |
||||
t := db.zsetBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
if zcnt, err := db.ZCard(key); err != nil || zcnt == 0 { |
||||
return 0, err |
||||
} else { |
||||
db.expireAt(t, ZSetType, key, when) |
||||
if err := t.Commit(); err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
return 1, nil |
||||
} |
||||
|
||||
func (db *DB) ZAdd(key []byte, args ...ScorePair) (int64, error) { |
||||
if len(args) == 0 { |
||||
return 0, nil |
||||
} |
||||
|
||||
t := db.zsetBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
var num int64 = 0 |
||||
for i := 0; i < len(args); i++ { |
||||
score := args[i].Score |
||||
member := args[i].Member |
||||
|
||||
if err := checkZSetKMSize(key, member); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
if n, err := db.zSetItem(t, key, score, member); err != nil { |
||||
return 0, err |
||||
} else if n == 0 { |
||||
//add new
|
||||
num++ |
||||
} |
||||
} |
||||
|
||||
if _, err := db.zIncrSize(t, key, num); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
//todo add binlog
|
||||
err := t.Commit() |
||||
return num, err |
||||
} |
||||
|
||||
func (db *DB) zIncrSize(t *batch, key []byte, delta int64) (int64, error) { |
||||
sk := db.zEncodeSizeKey(key) |
||||
|
||||
size, err := Int64(db.bucket.Get(sk)) |
||||
if err != nil { |
||||
return 0, err |
||||
} else { |
||||
size += delta |
||||
if size <= 0 { |
||||
size = 0 |
||||
t.Delete(sk) |
||||
db.rmExpire(t, ZSetType, key) |
||||
} else { |
||||
t.Put(sk, PutInt64(size)) |
||||
} |
||||
} |
||||
|
||||
return size, nil |
||||
} |
||||
|
||||
func (db *DB) ZCard(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
sk := db.zEncodeSizeKey(key) |
||||
return Int64(db.bucket.Get(sk)) |
||||
} |
||||
|
||||
func (db *DB) ZScore(key []byte, member []byte) (int64, error) { |
||||
if err := checkZSetKMSize(key, member); err != nil { |
||||
return InvalidScore, err |
||||
} |
||||
|
||||
var score int64 = InvalidScore |
||||
|
||||
k := db.zEncodeSetKey(key, member) |
||||
if v, err := db.bucket.Get(k); err != nil { |
||||
return InvalidScore, err |
||||
} else if v == nil { |
||||
return InvalidScore, ErrScoreMiss |
||||
} else { |
||||
if score, err = Int64(v, nil); err != nil { |
||||
return InvalidScore, err |
||||
} |
||||
} |
||||
|
||||
return score, nil |
||||
} |
||||
|
||||
func (db *DB) ZRem(key []byte, members ...[]byte) (int64, error) { |
||||
if len(members) == 0 { |
||||
return 0, nil |
||||
} |
||||
|
||||
t := db.zsetBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
var num int64 = 0 |
||||
for i := 0; i < len(members); i++ { |
||||
if err := checkZSetKMSize(key, members[i]); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
if n, err := db.zDelItem(t, key, members[i], false); err != nil { |
||||
return 0, err |
||||
} else if n == 1 { |
||||
num++ |
||||
} |
||||
} |
||||
|
||||
if _, err := db.zIncrSize(t, key, -num); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
err := t.Commit() |
||||
return num, err |
||||
} |
||||
|
||||
func (db *DB) ZIncrBy(key []byte, delta int64, member []byte) (int64, error) { |
||||
if err := checkZSetKMSize(key, member); err != nil { |
||||
return InvalidScore, err |
||||
} |
||||
|
||||
t := db.zsetBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
ek := db.zEncodeSetKey(key, member) |
||||
|
||||
var oldScore int64 = 0 |
||||
v, err := db.bucket.Get(ek) |
||||
if err != nil { |
||||
return InvalidScore, err |
||||
} else if v == nil { |
||||
db.zIncrSize(t, key, 1) |
||||
} else { |
||||
if oldScore, err = Int64(v, err); err != nil { |
||||
return InvalidScore, err |
||||
} |
||||
} |
||||
|
||||
newScore := oldScore + delta |
||||
if newScore >= MaxScore || newScore <= MinScore { |
||||
return InvalidScore, errScoreOverflow |
||||
} |
||||
|
||||
sk := db.zEncodeScoreKey(key, member, newScore) |
||||
t.Put(sk, []byte{}) |
||||
t.Put(ek, PutInt64(newScore)) |
||||
|
||||
if v != nil { |
||||
// so as to update score, we must delete the old one
|
||||
oldSk := db.zEncodeScoreKey(key, member, oldScore) |
||||
t.Delete(oldSk) |
||||
} |
||||
|
||||
err = t.Commit() |
||||
return newScore, err |
||||
} |
||||
|
||||
func (db *DB) ZCount(key []byte, min int64, max int64) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
minKey := db.zEncodeStartScoreKey(key, min) |
||||
maxKey := db.zEncodeStopScoreKey(key, max) |
||||
|
||||
rangeType := store.RangeROpen |
||||
|
||||
it := db.bucket.RangeLimitIterator(minKey, maxKey, rangeType, 0, -1) |
||||
var n int64 = 0 |
||||
for ; it.Valid(); it.Next() { |
||||
n++ |
||||
} |
||||
it.Close() |
||||
|
||||
return n, nil |
||||
} |
||||
|
||||
func (db *DB) zrank(key []byte, member []byte, reverse bool) (int64, error) { |
||||
if err := checkZSetKMSize(key, member); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
k := db.zEncodeSetKey(key, member) |
||||
|
||||
it := db.bucket.NewIterator() |
||||
defer it.Close() |
||||
|
||||
if v := it.Find(k); v == nil { |
||||
return -1, nil |
||||
} else { |
||||
if s, err := Int64(v, nil); err != nil { |
||||
return 0, err |
||||
} else { |
||||
var rit *store.RangeLimitIterator |
||||
|
||||
sk := db.zEncodeScoreKey(key, member, s) |
||||
|
||||
if !reverse { |
||||
minKey := db.zEncodeStartScoreKey(key, MinScore) |
||||
|
||||
rit = store.NewRangeIterator(it, &store.Range{minKey, sk, store.RangeClose}) |
||||
} else { |
||||
maxKey := db.zEncodeStopScoreKey(key, MaxScore) |
||||
rit = store.NewRevRangeIterator(it, &store.Range{sk, maxKey, store.RangeClose}) |
||||
} |
||||
|
||||
var lastKey []byte = nil |
||||
var n int64 = 0 |
||||
|
||||
for ; rit.Valid(); rit.Next() { |
||||
n++ |
||||
|
||||
lastKey = rit.BufKey(lastKey) |
||||
} |
||||
|
||||
if _, m, _, err := db.zDecodeScoreKey(lastKey); err == nil && bytes.Equal(m, member) { |
||||
n-- |
||||
return n, nil |
||||
} |
||||
} |
||||
} |
||||
|
||||
return -1, nil |
||||
} |
||||
|
||||
func (db *DB) zIterator(key []byte, min int64, max int64, offset int, count int, reverse bool) *store.RangeLimitIterator { |
||||
minKey := db.zEncodeStartScoreKey(key, min) |
||||
maxKey := db.zEncodeStopScoreKey(key, max) |
||||
|
||||
if !reverse { |
||||
return db.bucket.RangeLimitIterator(minKey, maxKey, store.RangeClose, offset, count) |
||||
} else { |
||||
return db.bucket.RevRangeLimitIterator(minKey, maxKey, store.RangeClose, offset, count) |
||||
} |
||||
} |
||||
|
||||
func (db *DB) zRemRange(t *batch, key []byte, min int64, max int64, offset int, count int) (int64, error) { |
||||
if len(key) > MaxKeySize { |
||||
return 0, errKeySize |
||||
} |
||||
|
||||
it := db.zIterator(key, min, max, offset, count, false) |
||||
var num int64 = 0 |
||||
for ; it.Valid(); it.Next() { |
||||
sk := it.RawKey() |
||||
_, m, _, err := db.zDecodeScoreKey(sk) |
||||
if err != nil { |
||||
continue |
||||
} |
||||
|
||||
if n, err := db.zDelItem(t, key, m, true); err != nil { |
||||
return 0, err |
||||
} else if n == 1 { |
||||
num++ |
||||
} |
||||
|
||||
t.Delete(sk) |
||||
} |
||||
it.Close() |
||||
|
||||
if _, err := db.zIncrSize(t, key, -num); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
return num, nil |
||||
} |
||||
|
||||
func (db *DB) zRange(key []byte, min int64, max int64, offset int, count int, reverse bool) ([]ScorePair, error) { |
||||
if len(key) > MaxKeySize { |
||||
return nil, errKeySize |
||||
} |
||||
|
||||
if offset < 0 { |
||||
return []ScorePair{}, nil |
||||
} |
||||
|
||||
nv := 64 |
||||
if count > 0 { |
||||
nv = count |
||||
} |
||||
|
||||
v := make([]ScorePair, 0, nv) |
||||
|
||||
var it *store.RangeLimitIterator |
||||
|
||||
//if reverse and offset is 0, count < 0, we may use forward iterator then reverse
|
||||
//because store iterator prev is slower than next
|
||||
if !reverse || (offset == 0 && count < 0) { |
||||
it = db.zIterator(key, min, max, offset, count, false) |
||||
} else { |
||||
it = db.zIterator(key, min, max, offset, count, true) |
||||
} |
||||
|
||||
for ; it.Valid(); it.Next() { |
||||
_, m, s, err := db.zDecodeScoreKey(it.Key()) |
||||
//may be we will check key equal?
|
||||
if err != nil { |
||||
continue |
||||
} |
||||
|
||||
v = append(v, ScorePair{Member: m, Score: s}) |
||||
} |
||||
it.Close() |
||||
|
||||
if reverse && (offset == 0 && count < 0) { |
||||
for i, j := 0, len(v)-1; i < j; i, j = i+1, j-1 { |
||||
v[i], v[j] = v[j], v[i] |
||||
} |
||||
} |
||||
|
||||
return v, nil |
||||
} |
||||
|
||||
func (db *DB) zParseLimit(key []byte, start int, stop int) (offset int, count int, err error) { |
||||
if start < 0 || stop < 0 { |
||||
//refer redis implementation
|
||||
var size int64 |
||||
size, err = db.ZCard(key) |
||||
if err != nil { |
||||
return |
||||
} |
||||
|
||||
llen := int(size) |
||||
|
||||
if start < 0 { |
||||
start = llen + start |
||||
} |
||||
if stop < 0 { |
||||
stop = llen + stop |
||||
} |
||||
|
||||
if start < 0 { |
||||
start = 0 |
||||
} |
||||
|
||||
if start >= llen { |
||||
offset = -1 |
||||
return |
||||
} |
||||
} |
||||
|
||||
if start > stop { |
||||
offset = -1 |
||||
return |
||||
} |
||||
|
||||
offset = start |
||||
count = (stop - start) + 1 |
||||
return |
||||
} |
||||
|
||||
func (db *DB) ZClear(key []byte) (int64, error) { |
||||
t := db.zsetBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
rmCnt, err := db.zRemRange(t, key, MinScore, MaxScore, 0, -1) |
||||
if err == nil { |
||||
err = t.Commit() |
||||
} |
||||
|
||||
return rmCnt, err |
||||
} |
||||
|
||||
func (db *DB) ZMclear(keys ...[]byte) (int64, error) { |
||||
t := db.zsetBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
for _, key := range keys { |
||||
if _, err := db.zRemRange(t, key, MinScore, MaxScore, 0, -1); err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
|
||||
err := t.Commit() |
||||
|
||||
return int64(len(keys)), err |
||||
} |
||||
|
||||
func (db *DB) ZRange(key []byte, start int, stop int) ([]ScorePair, error) { |
||||
return db.ZRangeGeneric(key, start, stop, false) |
||||
} |
||||
|
||||
//min and max must be inclusive
|
||||
//if no limit, set offset = 0 and count = -1
|
||||
func (db *DB) ZRangeByScore(key []byte, min int64, max int64, |
||||
offset int, count int) ([]ScorePair, error) { |
||||
return db.ZRangeByScoreGeneric(key, min, max, offset, count, false) |
||||
} |
||||
|
||||
func (db *DB) ZRank(key []byte, member []byte) (int64, error) { |
||||
return db.zrank(key, member, false) |
||||
} |
||||
|
||||
func (db *DB) ZRemRangeByRank(key []byte, start int, stop int) (int64, error) { |
||||
offset, count, err := db.zParseLimit(key, start, stop) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
var rmCnt int64 |
||||
|
||||
t := db.zsetBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
rmCnt, err = db.zRemRange(t, key, MinScore, MaxScore, offset, count) |
||||
if err == nil { |
||||
err = t.Commit() |
||||
} |
||||
|
||||
return rmCnt, err |
||||
} |
||||
|
||||
//min and max must be inclusive
|
||||
func (db *DB) ZRemRangeByScore(key []byte, min int64, max int64) (int64, error) { |
||||
t := db.zsetBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
rmCnt, err := db.zRemRange(t, key, min, max, 0, -1) |
||||
if err == nil { |
||||
err = t.Commit() |
||||
} |
||||
|
||||
return rmCnt, err |
||||
} |
||||
|
||||
func (db *DB) ZRevRange(key []byte, start int, stop int) ([]ScorePair, error) { |
||||
return db.ZRangeGeneric(key, start, stop, true) |
||||
} |
||||
|
||||
func (db *DB) ZRevRank(key []byte, member []byte) (int64, error) { |
||||
return db.zrank(key, member, true) |
||||
} |
||||
|
||||
//min and max must be inclusive
|
||||
//if no limit, set offset = 0 and count = -1
|
||||
func (db *DB) ZRevRangeByScore(key []byte, min int64, max int64, offset int, count int) ([]ScorePair, error) { |
||||
return db.ZRangeByScoreGeneric(key, min, max, offset, count, true) |
||||
} |
||||
|
||||
func (db *DB) ZRangeGeneric(key []byte, start int, stop int, reverse bool) ([]ScorePair, error) { |
||||
offset, count, err := db.zParseLimit(key, start, stop) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return db.zRange(key, MinScore, MaxScore, offset, count, reverse) |
||||
} |
||||
|
||||
//min and max must be inclusive
|
||||
//if no limit, set offset = 0 and count = -1
|
||||
func (db *DB) ZRangeByScoreGeneric(key []byte, min int64, max int64, |
||||
offset int, count int, reverse bool) ([]ScorePair, error) { |
||||
|
||||
return db.zRange(key, min, max, offset, count, reverse) |
||||
} |
||||
|
||||
func (db *DB) zFlush() (drop int64, err error) { |
||||
t := db.zsetBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
return db.flushType(t, ZSetType) |
||||
} |
||||
|
||||
func (db *DB) ZExpire(key []byte, duration int64) (int64, error) { |
||||
if duration <= 0 { |
||||
return 0, errExpireValue |
||||
} |
||||
|
||||
return db.zExpireAt(key, time.Now().Unix()+duration) |
||||
} |
||||
|
||||
func (db *DB) ZExpireAt(key []byte, when int64) (int64, error) { |
||||
if when <= time.Now().Unix() { |
||||
return 0, errExpireValue |
||||
} |
||||
|
||||
return db.zExpireAt(key, when) |
||||
} |
||||
|
||||
func (db *DB) ZTTL(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return -1, err |
||||
} |
||||
|
||||
return db.ttl(ZSetType, key) |
||||
} |
||||
|
||||
func (db *DB) ZPersist(key []byte) (int64, error) { |
||||
if err := checkKeySize(key); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
t := db.zsetBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
n, err := db.rmExpire(t, ZSetType, key) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
err = t.Commit() |
||||
return n, err |
||||
} |
||||
|
||||
func getAggregateFunc(aggregate byte) func(int64, int64) int64 { |
||||
switch aggregate { |
||||
case AggregateSum: |
||||
return func(a int64, b int64) int64 { |
||||
return a + b |
||||
} |
||||
case AggregateMax: |
||||
return func(a int64, b int64) int64 { |
||||
if a > b { |
||||
return a |
||||
} |
||||
return b |
||||
} |
||||
case AggregateMin: |
||||
return func(a int64, b int64) int64 { |
||||
if a > b { |
||||
return b |
||||
} |
||||
return a |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (db *DB) ZUnionStore(destKey []byte, srcKeys [][]byte, weights []int64, aggregate byte) (int64, error) { |
||||
|
||||
var destMap = map[string]int64{} |
||||
aggregateFunc := getAggregateFunc(aggregate) |
||||
if aggregateFunc == nil { |
||||
return 0, errInvalidAggregate |
||||
} |
||||
if len(srcKeys) < 1 { |
||||
return 0, errInvalidSrcKeyNum |
||||
} |
||||
if weights != nil { |
||||
if len(srcKeys) != len(weights) { |
||||
return 0, errInvalidWeightNum |
||||
} |
||||
} else { |
||||
weights = make([]int64, len(srcKeys)) |
||||
for i := 0; i < len(weights); i++ { |
||||
weights[i] = 1 |
||||
} |
||||
} |
||||
|
||||
for i, key := range srcKeys { |
||||
scorePairs, err := db.ZRange(key, 0, -1) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
for _, pair := range scorePairs { |
||||
if score, ok := destMap[String(pair.Member)]; !ok { |
||||
destMap[String(pair.Member)] = pair.Score * weights[i] |
||||
} else { |
||||
destMap[String(pair.Member)] = aggregateFunc(score, pair.Score*weights[i]) |
||||
} |
||||
} |
||||
} |
||||
|
||||
t := db.zsetBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
db.zDelete(t, destKey) |
||||
|
||||
for member, score := range destMap { |
||||
if err := checkZSetKMSize(destKey, []byte(member)); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
if _, err := db.zSetItem(t, destKey, score, []byte(member)); err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
|
||||
var num = int64(len(destMap)) |
||||
sk := db.zEncodeSizeKey(destKey) |
||||
t.Put(sk, PutInt64(num)) |
||||
|
||||
//todo add binlog
|
||||
if err := t.Commit(); err != nil { |
||||
return 0, err |
||||
} |
||||
return num, nil |
||||
} |
||||
|
||||
func (db *DB) ZInterStore(destKey []byte, srcKeys [][]byte, weights []int64, aggregate byte) (int64, error) { |
||||
|
||||
aggregateFunc := getAggregateFunc(aggregate) |
||||
if aggregateFunc == nil { |
||||
return 0, errInvalidAggregate |
||||
} |
||||
if len(srcKeys) < 1 { |
||||
return 0, errInvalidSrcKeyNum |
||||
} |
||||
if weights != nil { |
||||
if len(srcKeys) != len(weights) { |
||||
return 0, errInvalidWeightNum |
||||
} |
||||
} else { |
||||
weights = make([]int64, len(srcKeys)) |
||||
for i := 0; i < len(weights); i++ { |
||||
weights[i] = 1 |
||||
} |
||||
} |
||||
|
||||
var destMap = map[string]int64{} |
||||
scorePairs, err := db.ZRange(srcKeys[0], 0, -1) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
for _, pair := range scorePairs { |
||||
destMap[String(pair.Member)] = pair.Score * weights[0] |
||||
} |
||||
|
||||
for i, key := range srcKeys[1:] { |
||||
scorePairs, err := db.ZRange(key, 0, -1) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
tmpMap := map[string]int64{} |
||||
for _, pair := range scorePairs { |
||||
if score, ok := destMap[String(pair.Member)]; ok { |
||||
tmpMap[String(pair.Member)] = aggregateFunc(score, pair.Score*weights[i+1]) |
||||
} |
||||
} |
||||
destMap = tmpMap |
||||
} |
||||
|
||||
t := db.zsetBatch |
||||
t.Lock() |
||||
defer t.Unlock() |
||||
|
||||
db.zDelete(t, destKey) |
||||
|
||||
for member, score := range destMap { |
||||
if err := checkZSetKMSize(destKey, []byte(member)); err != nil { |
||||
return 0, err |
||||
} |
||||
if _, err := db.zSetItem(t, destKey, score, []byte(member)); err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
|
||||
var num int64 = int64(len(destMap)) |
||||
sk := db.zEncodeSizeKey(destKey) |
||||
t.Put(sk, PutInt64(num)) |
||||
//todo add binlog
|
||||
if err := t.Commit(); err != nil { |
||||
return 0, err |
||||
} |
||||
return num, nil |
||||
} |
||||
|
||||
func (db *DB) ZScan(key []byte, count int, inclusive bool, match string) ([][]byte, error) { |
||||
return db.scan(ZSizeType, key, count, inclusive, match) |
||||
} |
||||
@ -0,0 +1,113 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"errors" |
||||
"fmt" |
||||
|
||||
"github.com/lunny/nodb/store" |
||||
) |
||||
|
||||
var ( |
||||
ErrNestTx = errors.New("nest transaction not supported") |
||||
ErrTxDone = errors.New("Transaction has already been committed or rolled back") |
||||
) |
||||
|
||||
type Tx struct { |
||||
*DB |
||||
|
||||
tx *store.Tx |
||||
|
||||
logs [][]byte |
||||
} |
||||
|
||||
func (db *DB) IsTransaction() bool { |
||||
return db.status == DBInTransaction |
||||
} |
||||
|
||||
// Begin a transaction, it will block all other write operations before calling Commit or Rollback.
|
||||
// You must be very careful to prevent long-time transaction.
|
||||
func (db *DB) Begin() (*Tx, error) { |
||||
if db.IsTransaction() { |
||||
return nil, ErrNestTx |
||||
} |
||||
|
||||
tx := new(Tx) |
||||
|
||||
tx.DB = new(DB) |
||||
tx.DB.l = db.l |
||||
|
||||
tx.l.wLock.Lock() |
||||
|
||||
tx.DB.sdb = db.sdb |
||||
|
||||
var err error |
||||
tx.tx, err = db.sdb.Begin() |
||||
if err != nil { |
||||
tx.l.wLock.Unlock() |
||||
return nil, err |
||||
} |
||||
|
||||
tx.DB.bucket = tx.tx |
||||
|
||||
tx.DB.status = DBInTransaction |
||||
|
||||
tx.DB.index = db.index |
||||
|
||||
tx.DB.kvBatch = tx.newBatch() |
||||
tx.DB.listBatch = tx.newBatch() |
||||
tx.DB.hashBatch = tx.newBatch() |
||||
tx.DB.zsetBatch = tx.newBatch() |
||||
tx.DB.binBatch = tx.newBatch() |
||||
tx.DB.setBatch = tx.newBatch() |
||||
|
||||
return tx, nil |
||||
} |
||||
|
||||
func (tx *Tx) Commit() error { |
||||
if tx.tx == nil { |
||||
return ErrTxDone |
||||
} |
||||
|
||||
tx.l.commitLock.Lock() |
||||
err := tx.tx.Commit() |
||||
tx.tx = nil |
||||
|
||||
if len(tx.logs) > 0 { |
||||
tx.l.binlog.Log(tx.logs...) |
||||
} |
||||
|
||||
tx.l.commitLock.Unlock() |
||||
|
||||
tx.l.wLock.Unlock() |
||||
|
||||
tx.DB.bucket = nil |
||||
|
||||
return err |
||||
} |
||||
|
||||
func (tx *Tx) Rollback() error { |
||||
if tx.tx == nil { |
||||
return ErrTxDone |
||||
} |
||||
|
||||
err := tx.tx.Rollback() |
||||
tx.tx = nil |
||||
|
||||
tx.l.wLock.Unlock() |
||||
tx.DB.bucket = nil |
||||
|
||||
return err |
||||
} |
||||
|
||||
func (tx *Tx) newBatch() *batch { |
||||
return tx.l.newBatch(tx.tx.NewWriteBatch(), &txBatchLocker{}, tx) |
||||
} |
||||
|
||||
func (tx *Tx) Select(index int) error { |
||||
if index < 0 || index >= int(MaxDBNumber) { |
||||
return fmt.Errorf("invalid db index %d", index) |
||||
} |
||||
|
||||
tx.DB.index = uint8(index) |
||||
return nil |
||||
} |
||||
@ -0,0 +1,113 @@ |
||||
package nodb |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
"errors" |
||||
"reflect" |
||||
"strconv" |
||||
"unsafe" |
||||
) |
||||
|
||||
var errIntNumber = errors.New("invalid integer") |
||||
|
||||
// no copy to change slice to string
|
||||
// use your own risk
|
||||
func String(b []byte) (s string) { |
||||
pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) |
||||
pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) |
||||
pstring.Data = pbytes.Data |
||||
pstring.Len = pbytes.Len |
||||
return |
||||
} |
||||
|
||||
// no copy to change string to slice
|
||||
// use your own risk
|
||||
func Slice(s string) (b []byte) { |
||||
pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) |
||||
pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) |
||||
pbytes.Data = pstring.Data |
||||
pbytes.Len = pstring.Len |
||||
pbytes.Cap = pstring.Len |
||||
return |
||||
} |
||||
|
||||
func Int64(v []byte, err error) (int64, error) { |
||||
if err != nil { |
||||
return 0, err |
||||
} else if v == nil || len(v) == 0 { |
||||
return 0, nil |
||||
} else if len(v) != 8 { |
||||
return 0, errIntNumber |
||||
} |
||||
|
||||
return int64(binary.LittleEndian.Uint64(v)), nil |
||||
} |
||||
|
||||
func PutInt64(v int64) []byte { |
||||
var b []byte |
||||
pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) |
||||
pbytes.Data = uintptr(unsafe.Pointer(&v)) |
||||
pbytes.Len = 8 |
||||
pbytes.Cap = 8 |
||||
return b |
||||
} |
||||
|
||||
func StrInt64(v []byte, err error) (int64, error) { |
||||
if err != nil { |
||||
return 0, err |
||||
} else if v == nil { |
||||
return 0, nil |
||||
} else { |
||||
return strconv.ParseInt(String(v), 10, 64) |
||||
} |
||||
} |
||||
|
||||
func StrInt32(v []byte, err error) (int32, error) { |
||||
if err != nil { |
||||
return 0, err |
||||
} else if v == nil { |
||||
return 0, nil |
||||
} else { |
||||
res, err := strconv.ParseInt(String(v), 10, 32) |
||||
return int32(res), err |
||||
} |
||||
} |
||||
|
||||
func StrInt8(v []byte, err error) (int8, error) { |
||||
if err != nil { |
||||
return 0, err |
||||
} else if v == nil { |
||||
return 0, nil |
||||
} else { |
||||
res, err := strconv.ParseInt(String(v), 10, 8) |
||||
return int8(res), err |
||||
} |
||||
} |
||||
|
||||
func StrPutInt64(v int64) []byte { |
||||
return strconv.AppendInt(nil, v, 10) |
||||
} |
||||
|
||||
func MinUInt32(a uint32, b uint32) uint32 { |
||||
if a > b { |
||||
return b |
||||
} else { |
||||
return a |
||||
} |
||||
} |
||||
|
||||
func MaxUInt32(a uint32, b uint32) uint32 { |
||||
if a > b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
|
||||
func MaxInt32(a int32, b int32) int32 { |
||||
if a > b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
@ -0,0 +1,23 @@ |
||||
Copyright (c) 2015, Dave Cheney <dave@cheney.net> |
||||
All rights reserved. |
||||
|
||||
Redistribution and use in source and binary forms, with or without |
||||
modification, are permitted provided that the following conditions are met: |
||||
|
||||
* Redistributions of source code must retain the above copyright notice, this |
||||
list of conditions and the following disclaimer. |
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice, |
||||
this list of conditions and the following disclaimer in the documentation |
||||
and/or other materials provided with the distribution. |
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE |
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
@ -0,0 +1,282 @@ |
||||
// Package errors provides simple error handling primitives.
|
||||
//
|
||||
// The traditional error handling idiom in Go is roughly akin to
|
||||
//
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// which when applied recursively up the call stack results in error reports
|
||||
// without context or debugging information. The errors package allows
|
||||
// programmers to add context to the failure path in their code in a way
|
||||
// that does not destroy the original value of the error.
|
||||
//
|
||||
// Adding context to an error
|
||||
//
|
||||
// The errors.Wrap function returns a new error that adds context to the
|
||||
// original error by recording a stack trace at the point Wrap is called,
|
||||
// together with the supplied message. For example
|
||||
//
|
||||
// _, err := ioutil.ReadAll(r)
|
||||
// if err != nil {
|
||||
// return errors.Wrap(err, "read failed")
|
||||
// }
|
||||
//
|
||||
// If additional control is required, the errors.WithStack and
|
||||
// errors.WithMessage functions destructure errors.Wrap into its component
|
||||
// operations: annotating an error with a stack trace and with a message,
|
||||
// respectively.
|
||||
//
|
||||
// Retrieving the cause of an error
|
||||
//
|
||||
// Using errors.Wrap constructs a stack of errors, adding context to the
|
||||
// preceding error. Depending on the nature of the error it may be necessary
|
||||
// to reverse the operation of errors.Wrap to retrieve the original error
|
||||
// for inspection. Any error value which implements this interface
|
||||
//
|
||||
// type causer interface {
|
||||
// Cause() error
|
||||
// }
|
||||
//
|
||||
// can be inspected by errors.Cause. errors.Cause will recursively retrieve
|
||||
// the topmost error that does not implement causer, which is assumed to be
|
||||
// the original cause. For example:
|
||||
//
|
||||
// switch err := errors.Cause(err).(type) {
|
||||
// case *MyError:
|
||||
// // handle specifically
|
||||
// default:
|
||||
// // unknown error
|
||||
// }
|
||||
//
|
||||
// Although the causer interface is not exported by this package, it is
|
||||
// considered a part of its stable public interface.
|
||||
//
|
||||
// Formatted printing of errors
|
||||
//
|
||||
// All error values returned from this package implement fmt.Formatter and can
|
||||
// be formatted by the fmt package. The following verbs are supported:
|
||||
//
|
||||
// %s print the error. If the error has a Cause it will be
|
||||
// printed recursively.
|
||||
// %v see %s
|
||||
// %+v extended format. Each Frame of the error's StackTrace will
|
||||
// be printed in detail.
|
||||
//
|
||||
// Retrieving the stack trace of an error or wrapper
|
||||
//
|
||||
// New, Errorf, Wrap, and Wrapf record a stack trace at the point they are
|
||||
// invoked. This information can be retrieved with the following interface:
|
||||
//
|
||||
// type stackTracer interface {
|
||||
// StackTrace() errors.StackTrace
|
||||
// }
|
||||
//
|
||||
// The returned errors.StackTrace type is defined as
|
||||
//
|
||||
// type StackTrace []Frame
|
||||
//
|
||||
// The Frame type represents a call site in the stack trace. Frame supports
|
||||
// the fmt.Formatter interface that can be used for printing information about
|
||||
// the stack trace of this error. For example:
|
||||
//
|
||||
// if err, ok := err.(stackTracer); ok {
|
||||
// for _, f := range err.StackTrace() {
|
||||
// fmt.Printf("%+s:%d", f)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// Although the stackTracer interface is not exported by this package, it is
|
||||
// considered a part of its stable public interface.
|
||||
//
|
||||
// See the documentation for Frame.Format for more details.
|
||||
package errors |
||||
|
||||
import ( |
||||
"fmt" |
||||
"io" |
||||
) |
||||
|
||||
// New returns an error with the supplied message.
|
||||
// New also records the stack trace at the point it was called.
|
||||
func New(message string) error { |
||||
return &fundamental{ |
||||
msg: message, |
||||
stack: callers(), |
||||
} |
||||
} |
||||
|
||||
// Errorf formats according to a format specifier and returns the string
|
||||
// as a value that satisfies error.
|
||||
// Errorf also records the stack trace at the point it was called.
|
||||
func Errorf(format string, args ...interface{}) error { |
||||
return &fundamental{ |
||||
msg: fmt.Sprintf(format, args...), |
||||
stack: callers(), |
||||
} |
||||
} |
||||
|
||||
// fundamental is an error that has a message and a stack, but no caller.
|
||||
type fundamental struct { |
||||
msg string |
||||
*stack |
||||
} |
||||
|
||||
func (f *fundamental) Error() string { return f.msg } |
||||
|
||||
func (f *fundamental) Format(s fmt.State, verb rune) { |
||||
switch verb { |
||||
case 'v': |
||||
if s.Flag('+') { |
||||
io.WriteString(s, f.msg) |
||||
f.stack.Format(s, verb) |
||||
return |
||||
} |
||||
fallthrough |
||||
case 's': |
||||
io.WriteString(s, f.msg) |
||||
case 'q': |
||||
fmt.Fprintf(s, "%q", f.msg) |
||||
} |
||||
} |
||||
|
||||
// WithStack annotates err with a stack trace at the point WithStack was called.
|
||||
// If err is nil, WithStack returns nil.
|
||||
func WithStack(err error) error { |
||||
if err == nil { |
||||
return nil |
||||
} |
||||
return &withStack{ |
||||
err, |
||||
callers(), |
||||
} |
||||
} |
||||
|
||||
type withStack struct { |
||||
error |
||||
*stack |
||||
} |
||||
|
||||
func (w *withStack) Cause() error { return w.error } |
||||
|
||||
func (w *withStack) Format(s fmt.State, verb rune) { |
||||
switch verb { |
||||
case 'v': |
||||
if s.Flag('+') { |
||||
fmt.Fprintf(s, "%+v", w.Cause()) |
||||
w.stack.Format(s, verb) |
||||
return |
||||
} |
||||
fallthrough |
||||
case 's': |
||||
io.WriteString(s, w.Error()) |
||||
case 'q': |
||||
fmt.Fprintf(s, "%q", w.Error()) |
||||
} |
||||
} |
||||
|
||||
// Wrap returns an error annotating err with a stack trace
|
||||
// at the point Wrap is called, and the supplied message.
|
||||
// If err is nil, Wrap returns nil.
|
||||
func Wrap(err error, message string) error { |
||||
if err == nil { |
||||
return nil |
||||
} |
||||
err = &withMessage{ |
||||
cause: err, |
||||
msg: message, |
||||
} |
||||
return &withStack{ |
||||
err, |
||||
callers(), |
||||
} |
||||
} |
||||
|
||||
// Wrapf returns an error annotating err with a stack trace
|
||||
// at the point Wrapf is called, and the format specifier.
|
||||
// If err is nil, Wrapf returns nil.
|
||||
func Wrapf(err error, format string, args ...interface{}) error { |
||||
if err == nil { |
||||
return nil |
||||
} |
||||
err = &withMessage{ |
||||
cause: err, |
||||
msg: fmt.Sprintf(format, args...), |
||||
} |
||||
return &withStack{ |
||||
err, |
||||
callers(), |
||||
} |
||||
} |
||||
|
||||
// WithMessage annotates err with a new message.
|
||||
// If err is nil, WithMessage returns nil.
|
||||
func WithMessage(err error, message string) error { |
||||
if err == nil { |
||||
return nil |
||||
} |
||||
return &withMessage{ |
||||
cause: err, |
||||
msg: message, |
||||
} |
||||
} |
||||
|
||||
// WithMessagef annotates err with the format specifier.
|
||||
// If err is nil, WithMessagef returns nil.
|
||||
func WithMessagef(err error, format string, args ...interface{}) error { |
||||
if err == nil { |
||||
return nil |
||||
} |
||||
return &withMessage{ |
||||
cause: err, |
||||
msg: fmt.Sprintf(format, args...), |
||||
} |
||||
} |
||||
|
||||
type withMessage struct { |
||||
cause error |
||||
msg string |
||||
} |
||||
|
||||
func (w *withMessage) Error() string { return w.msg + ": " + w.cause.Error() } |
||||
func (w *withMessage) Cause() error { return w.cause } |
||||
|
||||
func (w *withMessage) Format(s fmt.State, verb rune) { |
||||
switch verb { |
||||
case 'v': |
||||
if s.Flag('+') { |
||||
fmt.Fprintf(s, "%+v\n", w.Cause()) |
||||
io.WriteString(s, w.msg) |
||||
return |
||||
} |
||||
fallthrough |
||||
case 's', 'q': |
||||
io.WriteString(s, w.Error()) |
||||
} |
||||
} |
||||
|
||||
// Cause returns the underlying cause of the error, if possible.
|
||||
// An error value has a cause if it implements the following
|
||||
// interface:
|
||||
//
|
||||
// type causer interface {
|
||||
// Cause() error
|
||||
// }
|
||||
//
|
||||
// If the error does not implement Cause, the original error will
|
||||
// be returned. If the error is nil, nil will be returned without further
|
||||
// investigation.
|
||||
func Cause(err error) error { |
||||
type causer interface { |
||||
Cause() error |
||||
} |
||||
|
||||
for err != nil { |
||||
cause, ok := err.(causer) |
||||
if !ok { |
||||
break |
||||
} |
||||
err = cause.Cause() |
||||
} |
||||
return err |
||||
} |
||||
@ -0,0 +1,147 @@ |
||||
package errors |
||||
|
||||
import ( |
||||
"fmt" |
||||
"io" |
||||
"path" |
||||
"runtime" |
||||
"strings" |
||||
) |
||||
|
||||
// Frame represents a program counter inside a stack frame.
|
||||
type Frame uintptr |
||||
|
||||
// pc returns the program counter for this frame;
|
||||
// multiple frames may have the same PC value.
|
||||
func (f Frame) pc() uintptr { return uintptr(f) - 1 } |
||||
|
||||
// file returns the full path to the file that contains the
|
||||
// function for this Frame's pc.
|
||||
func (f Frame) file() string { |
||||
fn := runtime.FuncForPC(f.pc()) |
||||
if fn == nil { |
||||
return "unknown" |
||||
} |
||||
file, _ := fn.FileLine(f.pc()) |
||||
return file |
||||
} |
||||
|
||||
// line returns the line number of source code of the
|
||||
// function for this Frame's pc.
|
||||
func (f Frame) line() int { |
||||
fn := runtime.FuncForPC(f.pc()) |
||||
if fn == nil { |
||||
return 0 |
||||
} |
||||
_, line := fn.FileLine(f.pc()) |
||||
return line |
||||
} |
||||
|
||||
// Format formats the frame according to the fmt.Formatter interface.
|
||||
//
|
||||
// %s source file
|
||||
// %d source line
|
||||
// %n function name
|
||||
// %v equivalent to %s:%d
|
||||
//
|
||||
// Format accepts flags that alter the printing of some verbs, as follows:
|
||||
//
|
||||
// %+s function name and path of source file relative to the compile time
|
||||
// GOPATH separated by \n\t (<funcname>\n\t<path>)
|
||||
// %+v equivalent to %+s:%d
|
||||
func (f Frame) Format(s fmt.State, verb rune) { |
||||
switch verb { |
||||
case 's': |
||||
switch { |
||||
case s.Flag('+'): |
||||
pc := f.pc() |
||||
fn := runtime.FuncForPC(pc) |
||||
if fn == nil { |
||||
io.WriteString(s, "unknown") |
||||
} else { |
||||
file, _ := fn.FileLine(pc) |
||||
fmt.Fprintf(s, "%s\n\t%s", fn.Name(), file) |
||||
} |
||||
default: |
||||
io.WriteString(s, path.Base(f.file())) |
||||
} |
||||
case 'd': |
||||
fmt.Fprintf(s, "%d", f.line()) |
||||
case 'n': |
||||
name := runtime.FuncForPC(f.pc()).Name() |
||||
io.WriteString(s, funcname(name)) |
||||
case 'v': |
||||
f.Format(s, 's') |
||||
io.WriteString(s, ":") |
||||
f.Format(s, 'd') |
||||
} |
||||
} |
||||
|
||||
// StackTrace is stack of Frames from innermost (newest) to outermost (oldest).
|
||||
type StackTrace []Frame |
||||
|
||||
// Format formats the stack of Frames according to the fmt.Formatter interface.
|
||||
//
|
||||
// %s lists source files for each Frame in the stack
|
||||
// %v lists the source file and line number for each Frame in the stack
|
||||
//
|
||||
// Format accepts flags that alter the printing of some verbs, as follows:
|
||||
//
|
||||
// %+v Prints filename, function, and line number for each Frame in the stack.
|
||||
func (st StackTrace) Format(s fmt.State, verb rune) { |
||||
switch verb { |
||||
case 'v': |
||||
switch { |
||||
case s.Flag('+'): |
||||
for _, f := range st { |
||||
fmt.Fprintf(s, "\n%+v", f) |
||||
} |
||||
case s.Flag('#'): |
||||
fmt.Fprintf(s, "%#v", []Frame(st)) |
||||
default: |
||||
fmt.Fprintf(s, "%v", []Frame(st)) |
||||
} |
||||
case 's': |
||||
fmt.Fprintf(s, "%s", []Frame(st)) |
||||
} |
||||
} |
||||
|
||||
// stack represents a stack of program counters.
|
||||
type stack []uintptr |
||||
|
||||
func (s *stack) Format(st fmt.State, verb rune) { |
||||
switch verb { |
||||
case 'v': |
||||
switch { |
||||
case st.Flag('+'): |
||||
for _, pc := range *s { |
||||
f := Frame(pc) |
||||
fmt.Fprintf(st, "\n%+v", f) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (s *stack) StackTrace() StackTrace { |
||||
f := make([]Frame, len(*s)) |
||||
for i := 0; i < len(f); i++ { |
||||
f[i] = Frame((*s)[i]) |
||||
} |
||||
return f |
||||
} |
||||
|
||||
func callers() *stack { |
||||
const depth = 32 |
||||
var pcs [depth]uintptr |
||||
n := runtime.Callers(3, pcs[:]) |
||||
var st stack = pcs[0:n] |
||||
return &st |
||||
} |
||||
|
||||
// funcname removes the path prefix component of a function's name reported by func.Name().
|
||||
func funcname(name string) string { |
||||
i := strings.LastIndex(name, "/") |
||||
name = name[i+1:] |
||||
i = strings.Index(name, ".") |
||||
return name[i+1:] |
||||
} |
||||
@ -0,0 +1,12 @@ |
||||
# This is the official list of Snappy-Go authors for copyright purposes. |
||||
# This file is distinct from the CONTRIBUTORS files. |
||||
# See the latter for an explanation. |
||||
|
||||
# Names should be added to this file as |
||||
# Name or Organization <email address> |
||||
# The email address is not required for organizations. |
||||
|
||||
# Please keep the list sorted. |
||||
|
||||
Google Inc. |
||||
Jan Mercl <0xjnml@gmail.com> |
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue