From a307c3ab25057325488f65ea882712a6308a0d47 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Mon, 12 Nov 2018 17:03:39 +0100 Subject: [PATCH 01/16] swarm/shed: initial implementation --- swarm/shed/internal/db.go | 84 +++++ swarm/shed/internal/db_test.go | 103 ++++++ swarm/shed/internal/field_json.go | 68 ++++ swarm/shed/internal/field_json_test.go | 125 ++++++++ swarm/shed/internal/field_string.go | 56 ++++ swarm/shed/internal/field_string_test.go | 108 +++++++ swarm/shed/internal/field_uint64.go | 91 ++++++ swarm/shed/internal/field_uint64_test.go | 188 +++++++++++ swarm/shed/internal/index.go | 204 ++++++++++++ swarm/shed/internal/index_test.go | 386 +++++++++++++++++++++++ swarm/shed/internal/schema.go | 115 +++++++ swarm/shed/internal/schema_test.go | 124 ++++++++ swarm/shed/shed.go | 247 +++++++++++++++ 13 files changed, 1899 insertions(+) create mode 100644 swarm/shed/internal/db.go create mode 100644 swarm/shed/internal/db_test.go create mode 100644 swarm/shed/internal/field_json.go create mode 100644 swarm/shed/internal/field_json_test.go create mode 100644 swarm/shed/internal/field_string.go create mode 100644 swarm/shed/internal/field_string_test.go create mode 100644 swarm/shed/internal/field_uint64.go create mode 100644 swarm/shed/internal/field_uint64_test.go create mode 100644 swarm/shed/internal/index.go create mode 100644 swarm/shed/internal/index_test.go create mode 100644 swarm/shed/internal/schema.go create mode 100644 swarm/shed/internal/schema_test.go create mode 100644 swarm/shed/shed.go diff --git a/swarm/shed/internal/db.go b/swarm/shed/internal/db.go new file mode 100644 index 00000000000..02c870e6252 --- /dev/null +++ b/swarm/shed/internal/db.go @@ -0,0 +1,84 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package internal + +import ( + "github.com/ethereum/go-ethereum/metrics" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/iterator" + "github.com/syndtr/goleveldb/leveldb/opt" +) + +const openFileLimit = 128 + +type DB struct { + ldb *leveldb.DB +} + +func NewDB(path string) (db *DB, err error) { + ldb, err := leveldb.OpenFile(path, &opt.Options{OpenFilesCacheCapacity: openFileLimit}) + if err != nil { + return nil, err + } + db = &DB{ldb: ldb} + + if _, err = db.getSchema(); err != nil { + if err == leveldb.ErrNotFound { + if err = db.putSchema(schema{ + Fields: make(map[string]fieldSpec), + Indexes: make(map[byte]indexSpec), + }); err != nil { + return nil, err + } + } else { + return nil, err + } + } + return db, nil +} + +func (db *DB) Put(key []byte, value []byte) (err error) { + metrics.GetOrRegisterCounter("DB.put", nil).Inc(1) + + return db.ldb.Put(key, value, nil) +} + +func (db *DB) Get(key []byte) (value []byte, err error) { + metrics.GetOrRegisterCounter("DB.get", nil).Inc(1) + + return db.ldb.Get(key, nil) +} + +func (db *DB) Delete(key []byte) error { + return db.ldb.Delete(key, nil) +} + +func (db *DB) NewIterator() iterator.Iterator { + metrics.GetOrRegisterCounter("DB.newiterator", nil).Inc(1) + + return db.ldb.NewIterator(nil, nil) +} + +func (db *DB) WriteBatch(batch *leveldb.Batch) error { + metrics.GetOrRegisterCounter("DB.write", nil).Inc(1) + + return db.ldb.Write(batch, nil) +} + +func (db *DB) Close() (err error) { + return db.ldb.Close() +} diff --git a/swarm/shed/internal/db_test.go b/swarm/shed/internal/db_test.go new file mode 100644 index 00000000000..ecb260bf712 --- /dev/null +++ b/swarm/shed/internal/db_test.go @@ -0,0 +1,103 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package internal + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestNewDB(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + s, err := db.getSchema() + if err != nil { + t.Fatal(err) + } + if s.Fields == nil { + t.Error("schema fields are empty") + } + if len(s.Fields) != 0 { + t.Errorf("got schema fields length %v, want %v", len(s.Fields), 0) + } + if s.Indexes == nil { + t.Error("schema indexes are empty") + } + if len(s.Indexes) != 0 { + t.Errorf("got schema indexes length %v, want %v", len(s.Indexes), 0) + } +} + +func TestDB_persistence(t *testing.T) { + dir, err := ioutil.TempDir("", "shed-test-persistence") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + db, err := NewDB(dir) + if err != nil { + t.Fatal(err) + } + stringField, err := db.NewStringField("preserve-me") + if err != nil { + t.Fatal(err) + } + want := "persistent value" + err = stringField.Put(want) + if err != nil { + t.Fatal(err) + } + err = db.Close() + if err != nil { + t.Fatal(err) + } + + db2, err := NewDB(dir) + if err != nil { + t.Fatal(err) + } + stringField2, err := db2.NewStringField("preserve-me") + if err != nil { + t.Fatal(err) + } + got, err := stringField2.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got string %q, want %q", got, want) + } +} + +func newTestDB(t *testing.T) (db *DB, cleanupFunc func()) { + t.Helper() + + dir, err := ioutil.TempDir("", "shed-test") + if err != nil { + t.Fatal(err) + } + cleanupFunc = func() { os.RemoveAll(dir) } + db, err = NewDB(dir) + if err != nil { + cleanupFunc() + t.Fatal(err) + } + return db, cleanupFunc +} diff --git a/swarm/shed/internal/field_json.go b/swarm/shed/internal/field_json.go new file mode 100644 index 00000000000..bbe634f0143 --- /dev/null +++ b/swarm/shed/internal/field_json.go @@ -0,0 +1,68 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package internal + +import ( + "encoding/json" + + "github.com/syndtr/goleveldb/leveldb" +) + +type JSONField struct { + db *DB + key []byte +} + +func (db *DB) NewJSONField(name string) (f JSONField, err error) { + key, err := db.schemaFieldKey(name, "json") + if err != nil { + return f, err + } + return JSONField{ + db: db, + key: key, + }, nil +} + +func (f JSONField) Unmarshal(val interface{}) (err error) { + b, err := f.db.Get(f.key) + if err != nil { + // Q: should we ignore not found + // if err == leveldb.ErrNotFound { + // return nil + // } + return err + } + return json.Unmarshal(b, val) +} + +func (f JSONField) Put(val interface{}) (err error) { + b, err := json.Marshal(val) + if err != nil { + return err + } + return f.db.Put(f.key, b) +} + +func (f JSONField) PutInBatch(batch *leveldb.Batch, val interface{}) (err error) { + b, err := json.Marshal(val) + if err != nil { + return err + } + batch.Put(f.key, b) + return nil +} diff --git a/swarm/shed/internal/field_json_test.go b/swarm/shed/internal/field_json_test.go new file mode 100644 index 00000000000..3465f7efbfc --- /dev/null +++ b/swarm/shed/internal/field_json_test.go @@ -0,0 +1,125 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package internal + +import ( + "testing" + + "github.com/syndtr/goleveldb/leveldb" +) + +func TestJSONField(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + complexField, err := db.NewJSONField("complex-field") + if err != nil { + t.Fatal(err) + } + + type complexStructure struct { + A string + } + + t.Run("unmarshal empty", func(t *testing.T) { + var s complexStructure + err := complexField.Unmarshal(&s) + if err != leveldb.ErrNotFound { + t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound) + } + want := "" + if s.A != want { + t.Errorf("got string %q, want %q", s.A, want) + } + }) + + t.Run("put", func(t *testing.T) { + want := complexStructure{ + A: "simple string value", + } + err = complexField.Put(want) + if err != nil { + t.Fatal(err) + } + var got complexStructure + err = complexField.Unmarshal(&got) + if err != nil { + t.Fatal(err) + } + if got.A != want.A { + t.Errorf("got string %q, want %q", got.A, want.A) + } + + t.Run("overwrite", func(t *testing.T) { + want := complexStructure{ + A: "overwritten string value", + } + err = complexField.Put(want) + if err != nil { + t.Fatal(err) + } + var got complexStructure + err = complexField.Unmarshal(&got) + if err != nil { + t.Fatal(err) + } + if got.A != want.A { + t.Errorf("got string %q, want %q", got.A, want.A) + } + }) + }) + + t.Run("put in batch", func(t *testing.T) { + batch := new(leveldb.Batch) + want := complexStructure{ + A: "simple string batch value", + } + complexField.PutInBatch(batch, want) + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + var got complexStructure + err := complexField.Unmarshal(&got) + if err != nil { + t.Fatal(err) + } + if got.A != want.A { + t.Errorf("got string %q, want %q", got, want) + } + + t.Run("overwrite", func(t *testing.T) { + batch := new(leveldb.Batch) + want := complexStructure{ + A: "overwritten string batch value", + } + complexField.PutInBatch(batch, want) + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + var got complexStructure + err := complexField.Unmarshal(&got) + if err != nil { + t.Fatal(err) + } + if got.A != want.A { + t.Errorf("got string %q, want %q", got, want) + } + }) + }) +} diff --git a/swarm/shed/internal/field_string.go b/swarm/shed/internal/field_string.go new file mode 100644 index 00000000000..18c8c840d48 --- /dev/null +++ b/swarm/shed/internal/field_string.go @@ -0,0 +1,56 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package internal + +import ( + "github.com/syndtr/goleveldb/leveldb" +) + +type StringField struct { + db *DB + key []byte +} + +func (db *DB) NewStringField(name string) (f StringField, err error) { + key, err := db.schemaFieldKey(name, "string") + if err != nil { + return f, err + } + return StringField{ + db: db, + key: key, + }, nil +} + +func (f StringField) Get() (val string, err error) { + b, err := f.db.Get(f.key) + if err != nil { + if err == leveldb.ErrNotFound { + return "", nil + } + return "", err + } + return string(b), nil +} + +func (f StringField) Put(val string) (err error) { + return f.db.Put(f.key, []byte(val)) +} + +func (f StringField) PutInBatch(batch *leveldb.Batch, val string) { + batch.Put(f.key, []byte(val)) +} diff --git a/swarm/shed/internal/field_string_test.go b/swarm/shed/internal/field_string_test.go new file mode 100644 index 00000000000..58d7bdbf8d6 --- /dev/null +++ b/swarm/shed/internal/field_string_test.go @@ -0,0 +1,108 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package internal + +import ( + "testing" + + "github.com/syndtr/goleveldb/leveldb" +) + +func TestStringField(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + simpleString, err := db.NewStringField("simple-string") + if err != nil { + t.Fatal(err) + } + + t.Run("get empty", func(t *testing.T) { + got, err := simpleString.Get() + if err != nil { + t.Fatal(err) + } + want := "" + if got != want { + t.Errorf("got string %q, want %q", got, want) + } + }) + + t.Run("put", func(t *testing.T) { + want := "simple string value" + err = simpleString.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := simpleString.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got string %q, want %q", got, want) + } + + t.Run("overwrite", func(t *testing.T) { + want := "overwritten string value" + err = simpleString.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := simpleString.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got string %q, want %q", got, want) + } + }) + }) + + t.Run("put in batch", func(t *testing.T) { + batch := new(leveldb.Batch) + want := "simple string batch value" + simpleString.PutInBatch(batch, want) + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := simpleString.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got string %q, want %q", got, want) + } + + t.Run("overwrite", func(t *testing.T) { + batch := new(leveldb.Batch) + want := "overwritten string batch value" + simpleString.PutInBatch(batch, want) + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := simpleString.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got string %q, want %q", got, want) + } + }) + }) +} diff --git a/swarm/shed/internal/field_uint64.go b/swarm/shed/internal/field_uint64.go new file mode 100644 index 00000000000..3d92a09ee3f --- /dev/null +++ b/swarm/shed/internal/field_uint64.go @@ -0,0 +1,91 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package internal + +import ( + "encoding/binary" + + "github.com/syndtr/goleveldb/leveldb" +) + +type Uint64Field struct { + db *DB + key []byte +} + +func (db *DB) NewUint64Field(name string) (f Uint64Field, err error) { + key, err := db.schemaFieldKey(name, "uint64") + if err != nil { + return f, err + } + return Uint64Field{ + db: db, + key: key, + }, nil +} + +func (f Uint64Field) Get() (val uint64, err error) { + b, err := f.db.Get(f.key) + if err != nil { + if err == leveldb.ErrNotFound { + return 0, nil + } + return 0, err + } + return binary.BigEndian.Uint64(b), nil +} + +func (f Uint64Field) Put(val uint64) (err error) { + return f.db.Put(f.key, encodeUint64(val)) +} + +func (f Uint64Field) PutInBatch(batch *leveldb.Batch, val uint64) { + batch.Put(f.key, encodeUint64(val)) +} + +func (f Uint64Field) Inc() (val uint64, err error) { + val, err = f.Get() + if err != nil { + if err == leveldb.ErrNotFound { + val = 0 + } else { + return 0, err + } + } + val++ + return val, f.Put(val) +} + +func (f Uint64Field) IncInBatch(batch *leveldb.Batch) (val uint64, err error) { + val, err = f.Get() + if err != nil { + if err == leveldb.ErrNotFound { + val = 0 + } else { + return 0, err + } + } + val++ + f.PutInBatch(batch, val) + return val, nil +} + +func encodeUint64(val uint64) (b []byte) { + b = make([]byte, 8) + binary.BigEndian.PutUint64(b, val) + return b +} diff --git a/swarm/shed/internal/field_uint64_test.go b/swarm/shed/internal/field_uint64_test.go new file mode 100644 index 00000000000..436971d251f --- /dev/null +++ b/swarm/shed/internal/field_uint64_test.go @@ -0,0 +1,188 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package internal + +import ( + "testing" + + "github.com/syndtr/goleveldb/leveldb" +) + +func TestUint64Field(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + counter, err := db.NewUint64Field("counter") + if err != nil { + t.Fatal(err) + } + + t.Run("get empty", func(t *testing.T) { + got, err := counter.Get() + if err != nil { + t.Fatal(err) + } + var want uint64 = 0 + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + }) + + t.Run("put", func(t *testing.T) { + var want uint64 = 42 + err = counter.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := counter.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + + t.Run("overwrite", func(t *testing.T) { + var want uint64 = 84 + err = counter.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := counter.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + }) + }) + + t.Run("put in batch", func(t *testing.T) { + batch := new(leveldb.Batch) + var want uint64 = 42 + counter.PutInBatch(batch, want) + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := counter.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + + t.Run("overwrite", func(t *testing.T) { + batch := new(leveldb.Batch) + var want uint64 = 84 + counter.PutInBatch(batch, want) + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := counter.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + }) + }) +} + +func TestUint64Field_Inc(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + counter, err := db.NewUint64Field("counter") + if err != nil { + t.Fatal(err) + } + + var want uint64 = 1 + got, err := counter.Inc() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + + want = 2 + got, err = counter.Inc() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } +} + +func TestUint64Field_IncInBatch(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + counter, err := db.NewUint64Field("counter") + if err != nil { + t.Fatal(err) + } + + batch := new(leveldb.Batch) + var want uint64 = 1 + got, err := counter.IncInBatch(batch) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err = counter.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + + batch2 := new(leveldb.Batch) + want = 2 + got, err = counter.IncInBatch(batch2) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + err = db.WriteBatch(batch2) + if err != nil { + t.Fatal(err) + } + got, err = counter.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } +} diff --git a/swarm/shed/internal/index.go b/swarm/shed/internal/index.go new file mode 100644 index 00000000000..3d86c83aa51 --- /dev/null +++ b/swarm/shed/internal/index.go @@ -0,0 +1,204 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package internal + +import ( + "github.com/syndtr/goleveldb/leveldb" +) + +type IndexItem struct { + Hash []byte + Data []byte + AccessTimestamp int64 + StoreTimestamp int64 +} + +func (i IndexItem) Join(i2 IndexItem) (new IndexItem) { + if i.Hash == nil { + i.Hash = i2.Hash + } + if i.Data == nil { + i.Data = i2.Data + } + if i.AccessTimestamp == 0 { + i.AccessTimestamp = i2.AccessTimestamp + } + if i.StoreTimestamp == 0 { + i.StoreTimestamp = i2.StoreTimestamp + } + return i +} + +type Index struct { + db *DB + prefix []byte + encodeKeyFunc func(fields IndexItem) (key []byte, err error) + decodeKeyFunc func(key []byte) (e IndexItem, err error) + encodeValueFunc func(fields IndexItem) (value []byte, err error) + decodeValueFunc func(value []byte) (e IndexItem, err error) +} + +type IndexFuncs struct { + EncodeKey func(fields IndexItem) (key []byte, err error) + DecodeKey func(key []byte) (e IndexItem, err error) + EncodeValue func(fields IndexItem) (value []byte, err error) + DecodeValue func(value []byte) (e IndexItem, err error) +} + +func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) { + id, err := db.schemaIndexID(name) + if err != nil { + return f, err + } + prefix := []byte{id} + return Index{ + db: db, + prefix: prefix, + encodeKeyFunc: func(e IndexItem) (key []byte, err error) { + key, err = funcs.EncodeKey(e) + if err != nil { + return nil, err + } + return append(append(make([]byte, 0, len(key)+1), prefix...), key...), nil + }, + decodeKeyFunc: func(key []byte) (e IndexItem, err error) { + return funcs.DecodeKey(key[1:]) + }, + encodeValueFunc: funcs.EncodeValue, + decodeValueFunc: funcs.DecodeValue, + }, nil +} + +func (f Index) Get(keyFields IndexItem) (out IndexItem, err error) { + key, err := f.encodeKeyFunc(keyFields) + if err != nil { + return out, err + } + value, err := f.db.Get(key) + if err != nil { + return out, err + } + out, err = f.decodeValueFunc(value) + if err != nil { + return out, err + } + return out.Join(keyFields), nil +} + +func (f Index) Put(i IndexItem) (err error) { + key, err := f.encodeKeyFunc(i) + if err != nil { + return err + } + value, err := f.encodeValueFunc(i) + if err != nil { + return err + } + return f.db.Put(key, value) +} + +func (f Index) PutInBatch(batch *leveldb.Batch, i IndexItem) (err error) { + key, err := f.encodeKeyFunc(i) + if err != nil { + return err + } + value, err := f.encodeValueFunc(i) + if err != nil { + return err + } + batch.Put(key, value) + return nil +} + +func (f Index) Delete(keyFields IndexItem) (err error) { + key, err := f.encodeKeyFunc(keyFields) + if err != nil { + return err + } + return f.db.Delete(key) +} + +func (f Index) DeleteInBatch(batch *leveldb.Batch, keyFields IndexItem) (err error) { + key, err := f.encodeKeyFunc(keyFields) + if err != nil { + return err + } + batch.Delete(key) + return nil +} + +type IterFunc func(item IndexItem) (stop bool, err error) + +func (f Index) IterateAll(fn IterFunc) (err error) { + it := f.db.NewIterator() + defer it.Release() + + for ok := it.Seek(f.prefix); ok; ok = it.Next() { + key := it.Key() + if key[0] != f.prefix[0] { + break + } + keyIndexItem, err := f.decodeKeyFunc(key) + if err != nil { + return err + } + valueIndexItem, err := f.decodeValueFunc(it.Value()) + if err != nil { + return err + } + stop, err := fn(keyIndexItem.Join(valueIndexItem)) + if err != nil { + return err + } + if stop { + break + } + } + return it.Error() +} + +func (f Index) IterateFrom(start IndexItem, fn IterFunc) (err error) { + startKey, err := f.encodeKeyFunc(start) + if err != nil { + return err + } + it := f.db.NewIterator() + defer it.Release() + + for ok := it.Seek(startKey); ok; ok = it.Next() { + key := it.Key() + if key[0] != f.prefix[0] { + break + } + keyIndexItem, err := f.decodeKeyFunc(key) + if err != nil { + return err + } + valueIndexItem, err := f.decodeValueFunc(it.Value()) + if err != nil { + return err + } + stop, err := fn(keyIndexItem.Join(valueIndexItem)) + if err != nil { + return err + } + if stop { + break + } + } + return it.Error() +} diff --git a/swarm/shed/internal/index_test.go b/swarm/shed/internal/index_test.go new file mode 100644 index 00000000000..ebff5693810 --- /dev/null +++ b/swarm/shed/internal/index_test.go @@ -0,0 +1,386 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package internal + +import ( + "bytes" + "encoding/binary" + "fmt" + "sort" + "testing" + "time" + + "github.com/syndtr/goleveldb/leveldb" +) + +var retrievalIndexFuncs = IndexFuncs{ + EncodeKey: func(fields IndexItem) (key []byte, err error) { + return fields.Hash, nil + }, + DecodeKey: func(key []byte) (e IndexItem, err error) { + e.Hash = key + return e, nil + }, + EncodeValue: func(fields IndexItem) (value []byte, err error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) + value = append(b, fields.Data...) + return value, nil + }, + DecodeValue: func(value []byte) (e IndexItem, err error) { + e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) + e.Data = value[8:] + return e, nil + }, +} + +func TestIndex(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + index, err := db.NewIndex("retrieval", retrievalIndexFuncs) + if err != nil { + t.Fatal(err) + } + + t.Run("put", func(t *testing.T) { + want := IndexItem{ + Hash: []byte("put-hash"), + Data: []byte("DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + err = index.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(IndexItem{ + Hash: want.Hash, + }) + if err != nil { + t.Fatal(err) + } + checkIndexItem(t, got, want) + + t.Run("overwrite", func(t *testing.T) { + want := IndexItem{ + Hash: []byte("put-hash"), + Data: []byte("New DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + err = index.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(IndexItem{ + Hash: want.Hash, + }) + if err != nil { + t.Fatal(err) + } + checkIndexItem(t, got, want) + }) + }) + + t.Run("put in batch", func(t *testing.T) { + want := IndexItem{ + Hash: []byte("put-in-batch-hash"), + Data: []byte("DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + batch := new(leveldb.Batch) + index.PutInBatch(batch, want) + db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(IndexItem{ + Hash: want.Hash, + }) + if err != nil { + t.Fatal(err) + } + checkIndexItem(t, got, want) + + t.Run("overwrite", func(t *testing.T) { + want := IndexItem{ + Hash: []byte("put-in-batch-hash"), + Data: []byte("New DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + batch := new(leveldb.Batch) + index.PutInBatch(batch, want) + db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(IndexItem{ + Hash: want.Hash, + }) + if err != nil { + t.Fatal(err) + } + checkIndexItem(t, got, want) + }) + }) + + t.Run("delete", func(t *testing.T) { + want := IndexItem{ + Hash: []byte("delete-hash"), + Data: []byte("DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + err = index.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(IndexItem{ + Hash: want.Hash, + }) + if err != nil { + t.Fatal(err) + } + checkIndexItem(t, got, want) + + err = index.Delete(IndexItem{ + Hash: want.Hash, + }) + if err != nil { + t.Fatal(err) + } + + got, err = index.Get(IndexItem{ + Hash: want.Hash, + }) + if err != leveldb.ErrNotFound { + t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound) + } + }) + + t.Run("delete in batch", func(t *testing.T) { + want := IndexItem{ + Hash: []byte("delete-in-batch-hash"), + Data: []byte("DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + + err = index.Put(want) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(IndexItem{ + Hash: want.Hash, + }) + if err != nil { + t.Fatal(err) + } + checkIndexItem(t, got, want) + + batch := new(leveldb.Batch) + index.DeleteInBatch(batch, IndexItem{ + Hash: want.Hash, + }) + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + + got, err = index.Get(IndexItem{ + Hash: want.Hash, + }) + if err != leveldb.ErrNotFound { + t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound) + } + }) +} + +func TestIndex_iterate(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + index, err := db.NewIndex("retrieval", retrievalIndexFuncs) + if err != nil { + t.Fatal(err) + } + + items := []IndexItem{ + { + Hash: []byte("iterate-hash-01"), + Data: []byte("data80"), + }, + { + Hash: []byte("iterate-hash-03"), + Data: []byte("data22"), + }, + { + Hash: []byte("iterate-hash-05"), + Data: []byte("data41"), + }, + { + Hash: []byte("iterate-hash-02"), + Data: []byte("data84"), + }, + { + Hash: []byte("iterate-hash-06"), + Data: []byte("data1"), + }, + } + batch := new(leveldb.Batch) + for _, i := range items { + index.PutInBatch(batch, i) + } + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + item04 := IndexItem{ + Hash: []byte("iterate-hash-04"), + Data: []byte("data0"), + } + err = index.Put(item04) + if err != nil { + t.Fatal(err) + } + items = append(items, item04) + + sort.SliceStable(items, func(i, j int) bool { + return bytes.Compare(items[i].Hash, items[j].Hash) < 0 + }) + + t.Run("all", func(t *testing.T) { + var i int + err := index.IterateAll(func(item IndexItem) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkIndexItem(t, item, want) + i++ + return false, nil + }) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("from", func(t *testing.T) { + startIndex := 2 + i := startIndex + err := index.IterateFrom(items[startIndex], func(item IndexItem) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkIndexItem(t, item, want) + i++ + return false, nil + }) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("stop", func(t *testing.T) { + var i int + stopIndex := 3 + var count int + err := index.IterateAll(func(item IndexItem) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkIndexItem(t, item, want) + count++ + if i == stopIndex { + return true, nil + } + i++ + return false, nil + }) + if err != nil { + t.Fatal(err) + } + wantItemsCount := stopIndex + 1 + if count != wantItemsCount { + t.Errorf("got %v items, expected %v", count, wantItemsCount) + } + }) + + t.Run("no overflow", func(t *testing.T) { + secondIndex, err := db.NewIndex("second-index", retrievalIndexFuncs) + if err != nil { + t.Fatal(err) + } + + secondIndexItem := IndexItem{ + Hash: []byte("iterate-hash-10"), + Data: []byte("data-second"), + } + err = secondIndex.Put(secondIndexItem) + if err != nil { + t.Fatal(err) + } + + var i int + err = index.IterateAll(func(item IndexItem) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkIndexItem(t, item, want) + i++ + return false, nil + }) + if err != nil { + t.Fatal(err) + } + + i = 0 + err = secondIndex.IterateAll(func(item IndexItem) (stop bool, err error) { + if i > 1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + checkIndexItem(t, item, secondIndexItem) + i++ + return false, nil + }) + if err != nil { + t.Fatal(err) + } + }) +} + +func checkIndexItem(t *testing.T, got, want IndexItem) { + t.Helper() + + if !bytes.Equal(got.Hash, want.Hash) { + t.Errorf("got hash %q, expected %q", string(got.Hash), string(want.Hash)) + } + if !bytes.Equal(got.Data, want.Data) { + t.Errorf("got data %q, expected %q", string(got.Data), string(want.Data)) + } + if got.StoreTimestamp != want.StoreTimestamp { + t.Errorf("got store timestamp %v, expected %v", got.StoreTimestamp, want.StoreTimestamp) + } + if got.AccessTimestamp != want.AccessTimestamp { + t.Errorf("got access timestamp %v, expected %v", got.AccessTimestamp, want.AccessTimestamp) + } +} diff --git a/swarm/shed/internal/schema.go b/swarm/shed/internal/schema.go new file mode 100644 index 00000000000..9440ba89558 --- /dev/null +++ b/swarm/shed/internal/schema.go @@ -0,0 +1,115 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package internal + +import ( + "encoding/json" + "errors" + "fmt" +) + +var ( + keySchema = []byte{0} + keyPrefixFields byte = 1 + keyPrefixIndexStart byte = 2 // Q: or maybe 7, to have more space for potential specific perfixes +) + +type schema struct { + Fields map[string]fieldSpec `json:"fields"` + Indexes map[byte]indexSpec `json:"indexes"` +} + +type fieldSpec struct { + Type string `json:"type"` +} + +type indexSpec struct { + Name string `json:"name"` +} + +func (db *DB) schemaFieldKey(name, fieldType string) (key []byte, err error) { + if name == "" { + return nil, errors.New("filed name can not be blank") + } + if fieldType == "" { + return nil, errors.New("filed type can not be blank") + } + s, err := db.getSchema() + if err != nil { + return nil, err + } + var found bool + for n, f := range s.Fields { + if n == name { + if f.Type != fieldType { + return nil, fmt.Errorf("field %q of type %q stored as %q in db", name, fieldType, f.Type) + } + break + } + } + if !found { + s.Fields[name] = fieldSpec{ + Type: fieldType, + } + err := db.putSchema(s) + if err != nil { + return nil, err + } + } + return append([]byte{keyPrefixFields}, []byte(name)...), nil +} + +func (db *DB) schemaIndexID(name string) (id byte, err error) { + if name == "" { + return 0, errors.New("index name can not be blank") + } + s, err := db.getSchema() + if err != nil { + return 0, err + } + nextID := keyPrefixIndexStart + for i, f := range s.Indexes { + if i >= nextID { + nextID = i + 1 + } + if f.Name == name { + return i, nil + } + } + id = nextID + s.Indexes[id] = indexSpec{ + Name: name, + } + return id, db.putSchema(s) +} + +func (db *DB) getSchema() (s schema, err error) { + b, err := db.Get(keySchema) + if err != nil { + return s, err + } + err = json.Unmarshal(b, &s) + return s, err +} + +func (db *DB) putSchema(s schema) (err error) { + b, err := json.Marshal(s) + if err != nil { + return err + } + return db.Put(keySchema, b) +} diff --git a/swarm/shed/internal/schema_test.go b/swarm/shed/internal/schema_test.go new file mode 100644 index 00000000000..8b3682a8e95 --- /dev/null +++ b/swarm/shed/internal/schema_test.go @@ -0,0 +1,124 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package internal + +import ( + "bytes" + "testing" +) + +func TestSchema_schemaFieldKey(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + t.Run("empty name or type", func(t *testing.T) { + _, err := db.schemaFieldKey("", "") + if err == nil { + t.Errorf("error not returned, but expected") + } + _, err = db.schemaFieldKey("", "type") + if err == nil { + t.Errorf("error not returned, but expected") + } + + _, err = db.schemaFieldKey("test", "") + if err == nil { + t.Errorf("error not returned, but expected") + } + }) + + t.Run("same field", func(t *testing.T) { + key1, err := db.schemaFieldKey("test", "undefined") + if err != nil { + t.Fatal(err) + } + + key2, err := db.schemaFieldKey("test", "undefined") + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(key1, key2) { + t.Errorf("schema keys for the same field name are not the same: %q, %q", string(key1), string(key2)) + } + }) + + t.Run("different fields", func(t *testing.T) { + key1, err := db.schemaFieldKey("test1", "undefined") + if err != nil { + t.Fatal(err) + } + + key2, err := db.schemaFieldKey("test2", "undefined") + if err != nil { + t.Fatal(err) + } + + if bytes.Equal(key1, key2) { + t.Error("schema keys for the same field name are the same, but must not be") + } + }) + + t.Run("same field name different types", func(t *testing.T) { + _, err := db.schemaFieldKey("the-field", "one-type") + if err != nil { + t.Fatal(err) + } + + _, err = db.schemaFieldKey("the-field", "another-type") + if err == nil { + t.Errorf("error not returned, but expected") + } + }) +} + +func TestSchema_schemaIndexID(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + t.Run("same name", func(t *testing.T) { + id1, err := db.schemaIndexID("test") + if err != nil { + t.Fatal(err) + } + + id2, err := db.schemaIndexID("test") + if err != nil { + t.Fatal(err) + } + + if id1 != id2 { + t.Errorf("schema keys for the same field name are not the same: %v, %v", id1, id2) + } + }) + + t.Run("different names", func(t *testing.T) { + id1, err := db.schemaIndexID("test1") + if err != nil { + t.Fatal(err) + } + + id2, err := db.schemaIndexID("test2") + if err != nil { + t.Fatal(err) + } + + if id1 == id2 { + t.Error("schema ids for the same index name are the same, but must not be") + } + }) +} diff --git a/swarm/shed/shed.go b/swarm/shed/shed.go new file mode 100644 index 00000000000..4f65d4ed814 --- /dev/null +++ b/swarm/shed/shed.go @@ -0,0 +1,247 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "context" + "encoding/binary" + "time" + + "github.com/ethereum/go-ethereum/swarm/shed/internal" + "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/syndtr/goleveldb/leveldb" +) + +// DB is just an example for composing indexes. +type DB struct { + db *internal.DB + + // fields and indexes + schemaName internal.StringField + sizeCounter internal.Uint64Field + accessCounter internal.Uint64Field + retrievalIndex internal.Index + accessIndex internal.Index + gcIndex internal.Index +} + +func New(path string) (db *DB, err error) { + idb, err := internal.NewDB(path) + if err != nil { + return nil, err + } + db = &DB{ + db: idb, + } + db.schemaName, err = idb.NewStringField("schema-name") + if err != nil { + return nil, err + } + db.sizeCounter, err = idb.NewUint64Field("size-counter") + if err != nil { + return nil, err + } + db.accessCounter, err = idb.NewUint64Field("access-counter") + if err != nil { + return nil, err + } + db.retrievalIndex, err = idb.NewIndex("Hash->StoreTimestamp|Data", internal.IndexFuncs{ + EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { + return fields.Hash, nil + }, + DecodeKey: func(key []byte) (e internal.IndexItem, err error) { + e.Hash = key + return e, nil + }, + EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) + value = append(b, fields.Data...) + return value, nil + }, + DecodeValue: func(value []byte) (e internal.IndexItem, err error) { + e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) + e.Data = value[8:] + return e, nil + }, + }) + db.accessIndex, err = idb.NewIndex("Hash->AccessTimestamp", internal.IndexFuncs{ + EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { + return fields.Hash, nil + }, + DecodeKey: func(key []byte) (e internal.IndexItem, err error) { + e.Hash = key + return e, nil + }, + EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp)) + return b, nil + }, + DecodeValue: func(value []byte) (e internal.IndexItem, err error) { + e.AccessTimestamp = int64(binary.BigEndian.Uint64(value)) + return e, nil + }, + }) + db.gcIndex, err = idb.NewIndex("AccessTimestamp|StoredTimestamp|Hash->nil", internal.IndexFuncs{ + EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { + b := make([]byte, 16, 16+len(fields.Hash)) + binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp)) + binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp)) + key = append(b, fields.Hash...) + return key, nil + }, + DecodeKey: func(key []byte) (e internal.IndexItem, err error) { + e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8])) + e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16])) + e.Hash = key[16:] + return e, nil + }, + EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { + return nil, nil + }, + DecodeValue: func(value []byte) (e internal.IndexItem, err error) { + return e, nil + }, + }) + if err != nil { + return nil, err + } + return db, nil +} + +func (db *DB) Put(_ context.Context, ch storage.Chunk) (err error) { + return db.retrievalIndex.Put(internal.IndexItem{ + Hash: ch.Address(), + Data: ch.Data(), + StoreTimestamp: time.Now().UTC().UnixNano(), + }) +} + +func (db *DB) Get(_ context.Context, ref storage.Address) (c storage.Chunk, err error) { + batch := new(leveldb.Batch) + + item, err := db.retrievalIndex.Get(internal.IndexItem{ + Hash: ref, + }) + if err != nil { + if err == leveldb.ErrNotFound { + return nil, storage.ErrChunkNotFound + } + return nil, err + } + + accessItem, err := db.accessIndex.Get(internal.IndexItem{ + Hash: ref, + }) + switch err { + case nil: + err = db.gcIndex.DeleteInBatch(batch, internal.IndexItem{ + Hash: item.Hash, + StoreTimestamp: accessItem.AccessTimestamp, + AccessTimestamp: item.StoreTimestamp, + }) + if err != nil { + return nil, err + } + case leveldb.ErrNotFound: + default: + return nil, err + } + + accessTimestamp := time.Now().UTC().UnixNano() + + err = db.accessIndex.PutInBatch(batch, internal.IndexItem{ + Hash: ref, + AccessTimestamp: accessTimestamp, + }) + if err != nil { + return nil, err + } + + err = db.gcIndex.PutInBatch(batch, internal.IndexItem{ + Hash: item.Hash, + AccessTimestamp: accessTimestamp, + StoreTimestamp: item.StoreTimestamp, + }) + if err != nil { + return nil, err + } + + err = db.db.WriteBatch(batch) + if err != nil { + return nil, err + } + + return storage.NewChunk(item.Hash, item.Data), nil +} + +func (db *DB) CollectGarbage() (err error) { + const maxTrashSize = 100 + maxRounds := 10 // adbitrary number, needs to be calculated + + for roundCount := 0; roundCount < maxRounds; roundCount++ { + var garbageCount int + trash := new(leveldb.Batch) + err = db.gcIndex.IterateAll(func(item internal.IndexItem) (stop bool, err error) { + err = db.retrievalIndex.DeleteInBatch(trash, item) + if err != nil { + return false, err + } + err = db.accessIndex.DeleteInBatch(trash, item) + if err != nil { + return false, err + } + err = db.gcIndex.DeleteInBatch(trash, item) + if err != nil { + return false, err + } + garbageCount++ + if garbageCount >= maxTrashSize { + return true, nil + } + return false, nil + }) + if err != nil { + return err + } + if garbageCount == 0 { + return nil + } + err = db.db.WriteBatch(trash) + if err != nil { + return err + } + } + return nil +} + +func (db *DB) GetSchema() (name string, err error) { + name, err = db.schemaName.Get() + if err == leveldb.ErrNotFound { + return "", nil + } + return name, err +} + +func (db *DB) PutSchema(name string) (err error) { + return db.schemaName.Put(name) +} + +func (db *DB) Close() { + db.db.Close() +} From 6d6afef9113e4efefded893055e488471c1c4844 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Wed, 14 Nov 2018 12:40:30 +0100 Subject: [PATCH 02/16] swarm/shed/internal: add example_dbstore_test --- swarm/shed/internal/example_dbstore_test.go | 280 ++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 swarm/shed/internal/example_dbstore_test.go diff --git a/swarm/shed/internal/example_dbstore_test.go b/swarm/shed/internal/example_dbstore_test.go new file mode 100644 index 00000000000..a9fd0372719 --- /dev/null +++ b/swarm/shed/internal/example_dbstore_test.go @@ -0,0 +1,280 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package internal_test + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "io/ioutil" + "log" + "os" + "time" + + "github.com/ethereum/go-ethereum/swarm/shed/internal" + "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/syndtr/goleveldb/leveldb" +) + +// DB is just an example for composing indexes. +type DB struct { + db *internal.DB + + // fields and indexes + schemaName internal.StringField + sizeCounter internal.Uint64Field + accessCounter internal.Uint64Field + retrievalIndex internal.Index + accessIndex internal.Index + gcIndex internal.Index +} + +func New(path string) (db *DB, err error) { + idb, err := internal.NewDB(path) + if err != nil { + return nil, err + } + db = &DB{ + db: idb, + } + db.schemaName, err = idb.NewStringField("schema-name") + if err != nil { + return nil, err + } + db.sizeCounter, err = idb.NewUint64Field("size-counter") + if err != nil { + return nil, err + } + db.accessCounter, err = idb.NewUint64Field("access-counter") + if err != nil { + return nil, err + } + db.retrievalIndex, err = idb.NewIndex("Hash->StoreTimestamp|Data", internal.IndexFuncs{ + EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { + return fields.Hash, nil + }, + DecodeKey: func(key []byte) (e internal.IndexItem, err error) { + e.Hash = key + return e, nil + }, + EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) + value = append(b, fields.Data...) + return value, nil + }, + DecodeValue: func(value []byte) (e internal.IndexItem, err error) { + e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) + e.Data = value[8:] + return e, nil + }, + }) + db.accessIndex, err = idb.NewIndex("Hash->AccessTimestamp", internal.IndexFuncs{ + EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { + return fields.Hash, nil + }, + DecodeKey: func(key []byte) (e internal.IndexItem, err error) { + e.Hash = key + return e, nil + }, + EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp)) + return b, nil + }, + DecodeValue: func(value []byte) (e internal.IndexItem, err error) { + e.AccessTimestamp = int64(binary.BigEndian.Uint64(value)) + return e, nil + }, + }) + db.gcIndex, err = idb.NewIndex("AccessTimestamp|StoredTimestamp|Hash->nil", internal.IndexFuncs{ + EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { + b := make([]byte, 16, 16+len(fields.Hash)) + binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp)) + binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp)) + key = append(b, fields.Hash...) + return key, nil + }, + DecodeKey: func(key []byte) (e internal.IndexItem, err error) { + e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8])) + e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16])) + e.Hash = key[16:] + return e, nil + }, + EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { + return nil, nil + }, + DecodeValue: func(value []byte) (e internal.IndexItem, err error) { + return e, nil + }, + }) + if err != nil { + return nil, err + } + return db, nil +} + +func (db *DB) Put(_ context.Context, ch storage.Chunk) (err error) { + return db.retrievalIndex.Put(internal.IndexItem{ + Hash: ch.Address(), + Data: ch.Data(), + StoreTimestamp: time.Now().UTC().UnixNano(), + }) +} + +func (db *DB) Get(_ context.Context, ref storage.Address) (c storage.Chunk, err error) { + batch := new(leveldb.Batch) + + item, err := db.retrievalIndex.Get(internal.IndexItem{ + Hash: ref, + }) + if err != nil { + if err == leveldb.ErrNotFound { + return nil, storage.ErrChunkNotFound + } + return nil, err + } + + accessItem, err := db.accessIndex.Get(internal.IndexItem{ + Hash: ref, + }) + switch err { + case nil: + err = db.gcIndex.DeleteInBatch(batch, internal.IndexItem{ + Hash: item.Hash, + StoreTimestamp: accessItem.AccessTimestamp, + AccessTimestamp: item.StoreTimestamp, + }) + if err != nil { + return nil, err + } + case leveldb.ErrNotFound: + default: + return nil, err + } + + accessTimestamp := time.Now().UTC().UnixNano() + + err = db.accessIndex.PutInBatch(batch, internal.IndexItem{ + Hash: ref, + AccessTimestamp: accessTimestamp, + }) + if err != nil { + return nil, err + } + + err = db.gcIndex.PutInBatch(batch, internal.IndexItem{ + Hash: item.Hash, + AccessTimestamp: accessTimestamp, + StoreTimestamp: item.StoreTimestamp, + }) + if err != nil { + return nil, err + } + + err = db.db.WriteBatch(batch) + if err != nil { + return nil, err + } + + return storage.NewChunk(item.Hash, item.Data), nil +} + +func (db *DB) CollectGarbage() (err error) { + const maxTrashSize = 100 + maxRounds := 10 // adbitrary number, needs to be calculated + + for roundCount := 0; roundCount < maxRounds; roundCount++ { + var garbageCount int + trash := new(leveldb.Batch) + err = db.gcIndex.IterateAll(func(item internal.IndexItem) (stop bool, err error) { + err = db.retrievalIndex.DeleteInBatch(trash, item) + if err != nil { + return false, err + } + err = db.accessIndex.DeleteInBatch(trash, item) + if err != nil { + return false, err + } + err = db.gcIndex.DeleteInBatch(trash, item) + if err != nil { + return false, err + } + garbageCount++ + if garbageCount >= maxTrashSize { + return true, nil + } + return false, nil + }) + if err != nil { + return err + } + if garbageCount == 0 { + return nil + } + err = db.db.WriteBatch(trash) + if err != nil { + return err + } + } + return nil +} + +func (db *DB) GetSchema() (name string, err error) { + name, err = db.schemaName.Get() + if err == leveldb.ErrNotFound { + return "", nil + } + return name, err +} + +func (db *DB) PutSchema(name string) (err error) { + return db.schemaName.Put(name) +} + +func (db *DB) Close() { + db.db.Close() +} + +func Example_dbstore() { + dir, err := ioutil.TempDir("", "ephemeral") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(dir) + + db, err := New(dir) + if err != nil { + log.Fatal(err) + } + + ch := storage.GenerateRandomChunk(1024) + err = db.Put(context.Background(), ch) + if err != nil { + log.Fatal(err) + } + + got, err := db.Get(context.Background(), ch.Address()) + if err != nil { + log.Fatal(err) + } + + fmt.Println(bytes.Equal(got.Data(), ch.Data())) + + //Output: true +} From a919a3214de1ad9dc73f82d6803b4169d4c8eece Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Wed, 14 Nov 2018 15:38:21 +0100 Subject: [PATCH 03/16] swarm/shed: remove internal package and add comments --- swarm/shed/{internal => }/db.go | 32 +- swarm/shed/{internal => }/db_test.go | 9 +- swarm/shed/example_store_test.go | 325 ++++++++++++++++++ swarm/shed/{internal => }/field_json.go | 14 +- swarm/shed/{internal => }/field_json_test.go | 4 +- swarm/shed/{internal => }/field_string.go | 12 +- .../shed/{internal => }/field_string_test.go | 4 +- swarm/shed/{internal => }/field_uint64.go | 19 +- .../shed/{internal => }/field_uint64_test.go | 10 +- swarm/shed/{internal => }/index.go | 62 +++- swarm/shed/{internal => }/index_test.go | 76 ++-- swarm/shed/internal/example_dbstore_test.go | 280 --------------- swarm/shed/{internal => }/schema.go | 33 +- swarm/shed/{internal => }/schema_test.go | 16 +- swarm/shed/shed.go | 247 ------------- 15 files changed, 543 insertions(+), 600 deletions(-) rename swarm/shed/{internal => }/db.go (62%) rename swarm/shed/{internal => }/db_test.go (85%) create mode 100644 swarm/shed/example_store_test.go rename swarm/shed/{internal => }/field_json.go (77%) rename swarm/shed/{internal => }/field_json_test.go (97%) rename swarm/shed/{internal => }/field_string.go (75%) rename swarm/shed/{internal => }/field_string_test.go (96%) rename swarm/shed/{internal => }/field_uint64.go (71%) rename swarm/shed/{internal => }/field_uint64_test.go (93%) rename swarm/shed/{internal => }/index.go (62%) rename swarm/shed/{internal => }/index_test.go (82%) delete mode 100644 swarm/shed/internal/example_dbstore_test.go rename swarm/shed/{internal => }/schema.go (63%) rename swarm/shed/{internal => }/schema_test.go (87%) delete mode 100644 swarm/shed/shed.go diff --git a/swarm/shed/internal/db.go b/swarm/shed/db.go similarity index 62% rename from swarm/shed/internal/db.go rename to swarm/shed/db.go index 02c870e6252..987c89dcff4 100644 --- a/swarm/shed/internal/db.go +++ b/swarm/shed/db.go @@ -14,7 +14,13 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +// Package shed provides a simple abstraction components to compose +// more complex operations on storage data organized in fields and indexes. +// +// Only type which holds logical information about swarm storage chunks data +// and metadata is IndexItem. This part is not generalized mostly for +// performance reasons. +package shed import ( "github.com/ethereum/go-ethereum/metrics" @@ -23,21 +29,33 @@ import ( "github.com/syndtr/goleveldb/leveldb/opt" ) +// The limit for LevelDB OpenFilesCacheCapacity. const openFileLimit = 128 +// DB provides abstractions over LevelDB in order to +// implement complex structures using fields and ordered indexes. +// It provides a schema functionality to store fields and indexes +// information about naming and types. type DB struct { ldb *leveldb.DB } +// NewDB constructs a new DB and validates the schema +// if it exists in database on the given path. func NewDB(path string) (db *DB, err error) { - ldb, err := leveldb.OpenFile(path, &opt.Options{OpenFilesCacheCapacity: openFileLimit}) + ldb, err := leveldb.OpenFile(path, &opt.Options{ + OpenFilesCacheCapacity: openFileLimit, + }) if err != nil { return nil, err } - db = &DB{ldb: ldb} + db = &DB{ + ldb: ldb, + } if _, err = db.getSchema(); err != nil { if err == leveldb.ErrNotFound { + // save schema with initialized default fields if err = db.putSchema(schema{ Fields: make(map[string]fieldSpec), Indexes: make(map[byte]indexSpec), @@ -51,34 +69,42 @@ func NewDB(path string) (db *DB, err error) { return db, nil } +// Put wraps LevelDB Put method to increment metrics counter. func (db *DB) Put(key []byte, value []byte) (err error) { metrics.GetOrRegisterCounter("DB.put", nil).Inc(1) return db.ldb.Put(key, value, nil) } +// Get wraps LevelDB Get method to increment metrics counter. func (db *DB) Get(key []byte) (value []byte, err error) { metrics.GetOrRegisterCounter("DB.get", nil).Inc(1) return db.ldb.Get(key, nil) } +// Delete wraps LevelDB Delete method to increment metrics counter. func (db *DB) Delete(key []byte) error { + metrics.GetOrRegisterCounter("DB.delete", nil).Inc(1) + return db.ldb.Delete(key, nil) } +// NewIterator wraps LevelDB NewIterator method to increment metrics counter. func (db *DB) NewIterator() iterator.Iterator { metrics.GetOrRegisterCounter("DB.newiterator", nil).Inc(1) return db.ldb.NewIterator(nil, nil) } +// WriteBatch wraps LevelDB Write method to increment metrics counter. func (db *DB) WriteBatch(batch *leveldb.Batch) error { metrics.GetOrRegisterCounter("DB.write", nil).Inc(1) return db.ldb.Write(batch, nil) } +// Close closes LevelDB database. func (db *DB) Close() (err error) { return db.ldb.Close() } diff --git a/swarm/shed/internal/db_test.go b/swarm/shed/db_test.go similarity index 85% rename from swarm/shed/internal/db_test.go rename to swarm/shed/db_test.go index ecb260bf712..45325beeb81 100644 --- a/swarm/shed/internal/db_test.go +++ b/swarm/shed/db_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "io/ioutil" @@ -22,6 +22,8 @@ import ( "testing" ) +// TestNewDB constructs a new DB +// and validates if the schema is initialized properly. func TestNewDB(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -44,6 +46,8 @@ func TestNewDB(t *testing.T) { } } +// TestDB_persistence creates one DB, saves a field and closes that DB. +// Then, it constructs another DB and trues to retrieve the saved value. func TestDB_persistence(t *testing.T) { dir, err := ioutil.TempDir("", "shed-test-persistence") if err != nil { @@ -86,6 +90,9 @@ func TestDB_persistence(t *testing.T) { } } +// newTestDB is a helper function that constructs a +// temporary database and returns a cleanup function that must +// be called to remove the data. func newTestDB(t *testing.T) (db *DB, cleanupFunc func()) { t.Helper() diff --git a/swarm/shed/example_store_test.go b/swarm/shed/example_store_test.go new file mode 100644 index 00000000000..733a972c65e --- /dev/null +++ b/swarm/shed/example_store_test.go @@ -0,0 +1,325 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed_test + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "io/ioutil" + "log" + "os" + "time" + + "github.com/ethereum/go-ethereum/swarm/shed" + "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/syndtr/goleveldb/leveldb" +) + +// Store holds fields and indexes (including their encoding functions) +// and defines operations on them by composing data from them. +// It implements storage.ChunkStore interface. +// It is just an example without any support for parallel operations +// or real world implementation. +type Store struct { + db *shed.DB + + // fields and indexes + schemaName shed.StringField + sizeCounter shed.Uint64Field + accessCounter shed.Uint64Field + retrievalIndex shed.Index + accessIndex shed.Index + gcIndex shed.Index +} + +// New returns new Store. All fields and indexes are initialized +// and possible conflicts with schema from existing database is checked +// automatically. +func New(path string) (s *Store, err error) { + db, err := shed.NewDB(path) + if err != nil { + return nil, err + } + s = &Store{ + db: db, + } + // Identify current storage schema by arbitrary name. + s.schemaName, err = db.NewStringField("schema-name") + if err != nil { + return nil, err + } + // Global ever incrementing index of chunk accesses. + s.accessCounter, err = db.NewUint64Field("access-counter") + if err != nil { + return nil, err + } + // Index storing actual chunk address, data and store timestamp. + s.retrievalIndex, err = db.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{ + EncodeKey: func(fields shed.IndexItem) (key []byte, err error) { + return fields.Address, nil + }, + DecodeKey: func(key []byte) (e shed.IndexItem, err error) { + e.Address = key + return e, nil + }, + EncodeValue: func(fields shed.IndexItem) (value []byte, err error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) + value = append(b, fields.Data...) + return value, nil + }, + DecodeValue: func(value []byte) (e shed.IndexItem, err error) { + e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) + e.Data = value[8:] + return e, nil + }, + }) + // Index storing access timestamp for a particular address. + // It is needed in order to update gc index keys for iteration order. + s.accessIndex, err = db.NewIndex("Address->AccessTimestamp", shed.IndexFuncs{ + EncodeKey: func(fields shed.IndexItem) (key []byte, err error) { + return fields.Address, nil + }, + DecodeKey: func(key []byte) (e shed.IndexItem, err error) { + e.Address = key + return e, nil + }, + EncodeValue: func(fields shed.IndexItem) (value []byte, err error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp)) + return b, nil + }, + DecodeValue: func(value []byte) (e shed.IndexItem, err error) { + e.AccessTimestamp = int64(binary.BigEndian.Uint64(value)) + return e, nil + }, + }) + // Index with keys ordered by access timestamp for garbage collection prioritization. + s.gcIndex, err = db.NewIndex("AccessTimestamp|StoredTimestamp|Address->nil", shed.IndexFuncs{ + EncodeKey: func(fields shed.IndexItem) (key []byte, err error) { + b := make([]byte, 16, 16+len(fields.Address)) + binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp)) + binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp)) + key = append(b, fields.Address...) + return key, nil + }, + DecodeKey: func(key []byte) (e shed.IndexItem, err error) { + e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8])) + e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16])) + e.Address = key[16:] + return e, nil + }, + EncodeValue: func(fields shed.IndexItem) (value []byte, err error) { + return nil, nil + }, + DecodeValue: func(value []byte) (e shed.IndexItem, err error) { + return e, nil + }, + }) + if err != nil { + return nil, err + } + return s, nil +} + +// Put stores the chunk and sets it store timestamp. +func (s *Store) Put(_ context.Context, ch storage.Chunk) (err error) { + return s.retrievalIndex.Put(shed.IndexItem{ + Address: ch.Address(), + Data: ch.Data(), + StoreTimestamp: time.Now().UTC().UnixNano(), + }) +} + +// Get retrieves a chunk with the provided address. +// It updates access and gc indexes by removing the previous +// items from them and adding new items as keys of index entries +// are changed. +func (s *Store) Get(_ context.Context, addr storage.Address) (c storage.Chunk, err error) { + batch := new(leveldb.Batch) + + // Get the chunk data and storage timestamp. + item, err := s.retrievalIndex.Get(shed.IndexItem{ + Address: addr, + }) + if err != nil { + if err == leveldb.ErrNotFound { + return nil, storage.ErrChunkNotFound + } + return nil, err + } + + // Get the chunk access timestamp. + accessItem, err := s.accessIndex.Get(shed.IndexItem{ + Address: addr, + }) + switch err { + case nil: + // Remove gc index entry if access timestamp is found. + err = s.gcIndex.DeleteInBatch(batch, shed.IndexItem{ + Address: item.Address, + StoreTimestamp: accessItem.AccessTimestamp, + AccessTimestamp: item.StoreTimestamp, + }) + if err != nil { + return nil, err + } + case leveldb.ErrNotFound: + // Access timestamp is not found. Do not do anything. + // This is the firs get request. + default: + return nil, err + } + + // Specify new access timestamp + accessTimestamp := time.Now().UTC().UnixNano() + + // Put new access timestamp in access index. + err = s.accessIndex.PutInBatch(batch, shed.IndexItem{ + Address: addr, + AccessTimestamp: accessTimestamp, + }) + if err != nil { + return nil, err + } + + // Put new access timestamp in gc index. + err = s.gcIndex.PutInBatch(batch, shed.IndexItem{ + Address: item.Address, + AccessTimestamp: accessTimestamp, + StoreTimestamp: item.StoreTimestamp, + }) + if err != nil { + return nil, err + } + + // Increment access counter. + // Currently this information is not used anywhere. + _, err = s.accessCounter.IncInBatch(batch) + if err != nil { + return nil, err + } + + // Write the batch. + err = s.db.WriteBatch(batch) + if err != nil { + return nil, err + } + + // Return the chunk. + return storage.NewChunk(item.Address, item.Data), nil +} + +// CollectGarbage is an example of index iteration. +// It provides no reliable garbage collection functionality. +func (s *Store) CollectGarbage() (err error) { + const maxTrashSize = 100 + maxRounds := 10 // arbitrary number, needs to be calculated + + // Run a few gc rounds. + for roundCount := 0; roundCount < maxRounds; roundCount++ { + var garbageCount int + // New batch for a new cg round. + trash := new(leveldb.Batch) + // Iterate through all index items and break when needed. + err = s.gcIndex.IterateAll(func(item shed.IndexItem) (stop bool, err error) { + // Remove the chunk. + err = s.retrievalIndex.DeleteInBatch(trash, item) + if err != nil { + return false, err + } + // Remove the element in gc index. + err = s.gcIndex.DeleteInBatch(trash, item) + if err != nil { + return false, err + } + // Remove the relation in access index. + err = s.accessIndex.DeleteInBatch(trash, item) + if err != nil { + return false, err + } + garbageCount++ + if garbageCount >= maxTrashSize { + return true, nil + } + return false, nil + }) + if err != nil { + return err + } + if garbageCount == 0 { + return nil + } + err = s.db.WriteBatch(trash) + if err != nil { + return err + } + } + return nil +} + +// GetSchema is an example of retrieveing the most simple +// string from a database field. +func (s *Store) GetSchema() (name string, err error) { + name, err = s.schemaName.Get() + if err == leveldb.ErrNotFound { + return "", nil + } + return name, err +} + +// GetSchema is an example of storing the most simple +// string in a database field. +func (s *Store) PutSchema(name string) (err error) { + return s.schemaName.Put(name) +} + +// Close closes the underlying database. +func (s *Store) Close() { + s.db.Close() +} + +// Example_store constructs a simple storage implementation using shed package. +func Example_store() { + dir, err := ioutil.TempDir("", "ephemeral") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(dir) + + s, err := New(dir) + if err != nil { + log.Fatal(err) + } + + ch := storage.GenerateRandomChunk(1024) + err = s.Put(context.Background(), ch) + if err != nil { + log.Fatal(err) + } + + got, err := s.Get(context.Background(), ch.Address()) + if err != nil { + log.Fatal(err) + } + + fmt.Println(bytes.Equal(got.Data(), ch.Data())) + + //Output: true +} diff --git a/swarm/shed/internal/field_json.go b/swarm/shed/field_json.go similarity index 77% rename from swarm/shed/internal/field_json.go rename to swarm/shed/field_json.go index bbe634f0143..b3274bb1b65 100644 --- a/swarm/shed/internal/field_json.go +++ b/swarm/shed/field_json.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "encoding/json" @@ -22,11 +22,15 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) +// JSONField is a helper to store complex structure by +// encoding it in JSON format. type JSONField struct { db *DB key []byte } +// NewJSONField returns a new JSONField. +// It validates its name and type against the database schema. func (db *DB) NewJSONField(name string) (f JSONField, err error) { key, err := db.schemaFieldKey(name, "json") if err != nil { @@ -38,18 +42,17 @@ func (db *DB) NewJSONField(name string) (f JSONField, err error) { }, nil } +// Unmarshal unmarshals data fromt he database to a provided val. +// If the data is not found leveldb.ErrNotFound is returned. func (f JSONField) Unmarshal(val interface{}) (err error) { b, err := f.db.Get(f.key) if err != nil { - // Q: should we ignore not found - // if err == leveldb.ErrNotFound { - // return nil - // } return err } return json.Unmarshal(b, val) } +// Put marshals provided val and saves it to the database. func (f JSONField) Put(val interface{}) (err error) { b, err := json.Marshal(val) if err != nil { @@ -58,6 +61,7 @@ func (f JSONField) Put(val interface{}) (err error) { return f.db.Put(f.key, b) } +// PutInBatch marshals provided val and puts it into the batch. func (f JSONField) PutInBatch(batch *leveldb.Batch, val interface{}) (err error) { b, err := json.Marshal(val) if err != nil { diff --git a/swarm/shed/internal/field_json_test.go b/swarm/shed/field_json_test.go similarity index 97% rename from swarm/shed/internal/field_json_test.go rename to swarm/shed/field_json_test.go index 3465f7efbfc..c9e2cf658d9 100644 --- a/swarm/shed/internal/field_json_test.go +++ b/swarm/shed/field_json_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "testing" @@ -22,6 +22,8 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) +// TestJSONField validates put and unmarshal operations +// of the JSONField. func TestJSONField(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() diff --git a/swarm/shed/internal/field_string.go b/swarm/shed/field_string.go similarity index 75% rename from swarm/shed/internal/field_string.go rename to swarm/shed/field_string.go index 18c8c840d48..c2e5ffa14a3 100644 --- a/swarm/shed/internal/field_string.go +++ b/swarm/shed/field_string.go @@ -14,17 +14,21 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "github.com/syndtr/goleveldb/leveldb" ) +// StringField is the most simple field implementation +// that stores an arbitrary string under a specific LevelDB key. type StringField struct { db *DB key []byte } +// NewStringField retruns a new Instance fo StringField. +// It validates its name and type against the database schema. func (db *DB) NewStringField(name string) (f StringField, err error) { key, err := db.schemaFieldKey(name, "string") if err != nil { @@ -36,6 +40,9 @@ func (db *DB) NewStringField(name string) (f StringField, err error) { }, nil } +// Get returns a string value from database. +// If the value is not found, an empty string is returned +// an no error. func (f StringField) Get() (val string, err error) { b, err := f.db.Get(f.key) if err != nil { @@ -47,10 +54,13 @@ func (f StringField) Get() (val string, err error) { return string(b), nil } +// Put stores a string in the database. func (f StringField) Put(val string) (err error) { return f.db.Put(f.key, []byte(val)) } +// PutInBatch stores a string in a batch that can be +// saved later in database. func (f StringField) PutInBatch(batch *leveldb.Batch, val string) { batch.Put(f.key, []byte(val)) } diff --git a/swarm/shed/internal/field_string_test.go b/swarm/shed/field_string_test.go similarity index 96% rename from swarm/shed/internal/field_string_test.go rename to swarm/shed/field_string_test.go index 58d7bdbf8d6..4215075bca6 100644 --- a/swarm/shed/internal/field_string_test.go +++ b/swarm/shed/field_string_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "testing" @@ -22,6 +22,8 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) +// TestStringField validates put and get operations +// of the StringField. func TestStringField(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() diff --git a/swarm/shed/internal/field_uint64.go b/swarm/shed/field_uint64.go similarity index 71% rename from swarm/shed/internal/field_uint64.go rename to swarm/shed/field_uint64.go index 3d92a09ee3f..80e0069ae43 100644 --- a/swarm/shed/internal/field_uint64.go +++ b/swarm/shed/field_uint64.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "encoding/binary" @@ -22,11 +22,15 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) +// Uint64Field provides a way to have a simple counter in the database. +// It transparently encodes uint64 type value to bytes. type Uint64Field struct { db *DB key []byte } +// NewUint64Field returns a new Uint64Field. +// It validates its name and type against the database schema. func (db *DB) NewUint64Field(name string) (f Uint64Field, err error) { key, err := db.schemaFieldKey(name, "uint64") if err != nil { @@ -38,6 +42,9 @@ func (db *DB) NewUint64Field(name string) (f Uint64Field, err error) { }, nil } +// Get retrieves a uint64 value from the database. +// If the value is not found in the database a 0 value +// is returned and no error. func (f Uint64Field) Get() (val uint64, err error) { b, err := f.db.Get(f.key) if err != nil { @@ -49,14 +56,19 @@ func (f Uint64Field) Get() (val uint64, err error) { return binary.BigEndian.Uint64(b), nil } +// Put encodes uin64 value and stores it in the database. func (f Uint64Field) Put(val uint64) (err error) { return f.db.Put(f.key, encodeUint64(val)) } +// PutInBatch stores a uint64 value in a batch +// that can be saved later in the database. func (f Uint64Field) PutInBatch(batch *leveldb.Batch, val uint64) { batch.Put(f.key, encodeUint64(val)) } +// Inc increments a uint64 value in the database. +// This operation is not goroutine save. func (f Uint64Field) Inc() (val uint64, err error) { val, err = f.Get() if err != nil { @@ -70,6 +82,9 @@ func (f Uint64Field) Inc() (val uint64, err error) { return val, f.Put(val) } +// IncInBatch increments a uint64 value in the batch +// by retreiving a value from the database, not the same batch. +// This operation is not goroutine save. func (f Uint64Field) IncInBatch(batch *leveldb.Batch) (val uint64, err error) { val, err = f.Get() if err != nil { @@ -84,6 +99,8 @@ func (f Uint64Field) IncInBatch(batch *leveldb.Batch) (val uint64, err error) { return val, nil } +// encode transforms uint64 to 8 byte long +// slice in big endian encoding. func encodeUint64(val uint64) (b []byte) { b = make([]byte, 8) binary.BigEndian.PutUint64(b, val) diff --git a/swarm/shed/internal/field_uint64_test.go b/swarm/shed/field_uint64_test.go similarity index 93% rename from swarm/shed/internal/field_uint64_test.go rename to swarm/shed/field_uint64_test.go index 436971d251f..69ade71ba3a 100644 --- a/swarm/shed/internal/field_uint64_test.go +++ b/swarm/shed/field_uint64_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "testing" @@ -22,6 +22,8 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) +// TestUint64Field validates put and get operations +// of the Uint64Field. func TestUint64Field(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -36,7 +38,7 @@ func TestUint64Field(t *testing.T) { if err != nil { t.Fatal(err) } - var want uint64 = 0 + var want uint64 if got != want { t.Errorf("got uint64 %v, want %v", got, want) } @@ -107,6 +109,8 @@ func TestUint64Field(t *testing.T) { }) } +// TestUint64Field_Inc validates Inc operation +// of the Uint64Field. func TestUint64Field_Inc(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -135,6 +139,8 @@ func TestUint64Field_Inc(t *testing.T) { } } +// TestUint64Field_IncInBatch validates IncInBatch operation +// of the Uint64Field. func TestUint64Field_IncInBatch(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() diff --git a/swarm/shed/internal/index.go b/swarm/shed/index.go similarity index 62% rename from swarm/shed/internal/index.go rename to swarm/shed/index.go index 3d86c83aa51..d1cf7a757f2 100644 --- a/swarm/shed/internal/index.go +++ b/swarm/shed/index.go @@ -14,22 +14,37 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "github.com/syndtr/goleveldb/leveldb" ) +// IndexItem holds fields relevant to Swarm Chunk data and metadata. +// All information required for swarm storage and operations +// on that storage must be defined here. +// This structure is logically connected to swarm storage, +// the only part of this package that is not generalized, +// mostly for performance reasons. +// +// IndexItem is a type that is used for retrieving, storing and encoding +// chunk data and metadata. It is passed as an argument to Index encoding +// functions, get function and put function. +// But it is also returned with additional data from get function call +// and as the argument in iterator function definition. type IndexItem struct { - Hash []byte + Address []byte Data []byte AccessTimestamp int64 StoreTimestamp int64 } +// Join is a helper method to construct a new +// IndexItem by filling up fields with default values +// of a particular IndexItem with values from another one. func (i IndexItem) Join(i2 IndexItem) (new IndexItem) { - if i.Hash == nil { - i.Hash = i2.Hash + if i.Address == nil { + i.Address = i2.Address } if i.Data == nil { i.Data = i2.Data @@ -43,6 +58,12 @@ func (i IndexItem) Join(i2 IndexItem) (new IndexItem) { return i } +// Index represents a set of LevelDB key value pairs that have common +// prefix. It holds functions for encoding and decoding keys and values +// to provide transparent actions on saved data which inclide: +// - getting a particular IndexItem +// - saving a particular IndexItem +// - iterating over a sorted LevelDB keys type Index struct { db *DB prefix []byte @@ -52,6 +73,8 @@ type Index struct { decodeValueFunc func(value []byte) (e IndexItem, err error) } +// IndexFuncs structure defines functions for encoding and decoding +// LevelDB keys and values for a specific index. type IndexFuncs struct { EncodeKey func(fields IndexItem) (key []byte, err error) DecodeKey func(key []byte) (e IndexItem, err error) @@ -59,8 +82,11 @@ type IndexFuncs struct { DecodeValue func(value []byte) (e IndexItem, err error) } +// NewIndex returns a new Index instance with defined name and +// encoding functions. The name must be unique and will be validated +// on database schema for a key prefix byte. func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) { - id, err := db.schemaIndexID(name) + id, err := db.schemaIndexPrefix(name) if err != nil { return f, err } @@ -83,6 +109,9 @@ func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) { }, nil } +// Get accepts key fields represented as IndexItem to retrieve a +// value from the index and return maximum available information +// from the index represented as another IndexItem. func (f Index) Get(keyFields IndexItem) (out IndexItem, err error) { key, err := f.encodeKeyFunc(keyFields) if err != nil { @@ -99,6 +128,8 @@ func (f Index) Get(keyFields IndexItem) (out IndexItem, err error) { return out.Join(keyFields), nil } +// Put accepts IndexItem to encode information from it +// and save it to the database. func (f Index) Put(i IndexItem) (err error) { key, err := f.encodeKeyFunc(i) if err != nil { @@ -111,6 +142,9 @@ func (f Index) Put(i IndexItem) (err error) { return f.db.Put(key, value) } +// PutInBatch is the same as Put method, but it just +// saves the key/value pair to the batch instead +// directly to the database. func (f Index) PutInBatch(batch *leveldb.Batch, i IndexItem) (err error) { key, err := f.encodeKeyFunc(i) if err != nil { @@ -124,6 +158,8 @@ func (f Index) PutInBatch(batch *leveldb.Batch, i IndexItem) (err error) { return nil } +// Delete accepts IndexItem to remove a key/value pair +// form the database based on its fields. func (f Index) Delete(keyFields IndexItem) (err error) { key, err := f.encodeKeyFunc(keyFields) if err != nil { @@ -132,6 +168,8 @@ func (f Index) Delete(keyFields IndexItem) (err error) { return f.db.Delete(key) } +// DeleteInBatch is the same as Delete just the operation +// is performed on the batch instead on the database. func (f Index) DeleteInBatch(batch *leveldb.Batch, keyFields IndexItem) (err error) { key, err := f.encodeKeyFunc(keyFields) if err != nil { @@ -141,9 +179,15 @@ func (f Index) DeleteInBatch(batch *leveldb.Batch, keyFields IndexItem) (err err return nil } -type IterFunc func(item IndexItem) (stop bool, err error) +// IndexIterFunc is a callback on every IndexItem that is decoded +// by iterating on an Index keys. +// By returning a true for stop variable, iteration will +// stop, and by returning the error, that error will be +// propagated to the called iterator method on Index. +type IndexIterFunc func(item IndexItem) (stop bool, err error) -func (f Index) IterateAll(fn IterFunc) (err error) { +// IterateAll iterates over all keys of the Index. +func (f Index) IterateAll(fn IndexIterFunc) (err error) { it := f.db.NewIterator() defer it.Release() @@ -171,7 +215,9 @@ func (f Index) IterateAll(fn IterFunc) (err error) { return it.Error() } -func (f Index) IterateFrom(start IndexItem, fn IterFunc) (err error) { +// IterateFrom iterates over Index keys starting from the key +// encoded from the provided IndexItem. +func (f Index) IterateFrom(start IndexItem, fn IndexIterFunc) (err error) { startKey, err := f.encodeKeyFunc(start) if err != nil { return err diff --git a/swarm/shed/internal/index_test.go b/swarm/shed/index_test.go similarity index 82% rename from swarm/shed/internal/index_test.go rename to swarm/shed/index_test.go index ebff5693810..4e3eba326a8 100644 --- a/swarm/shed/internal/index_test.go +++ b/swarm/shed/index_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "bytes" @@ -27,12 +27,13 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) +// Index functions for the index that is used in tests in this file. var retrievalIndexFuncs = IndexFuncs{ EncodeKey: func(fields IndexItem) (key []byte, err error) { - return fields.Hash, nil + return fields.Address, nil }, DecodeKey: func(key []byte) (e IndexItem, err error) { - e.Hash = key + e.Address = key return e, nil }, EncodeValue: func(fields IndexItem) (value []byte, err error) { @@ -48,6 +49,7 @@ var retrievalIndexFuncs = IndexFuncs{ }, } +// TestIndex validates put, get and delete functions of the index. func TestIndex(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -59,7 +61,7 @@ func TestIndex(t *testing.T) { t.Run("put", func(t *testing.T) { want := IndexItem{ - Hash: []byte("put-hash"), + Address: []byte("put-hash"), Data: []byte("DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), } @@ -69,7 +71,7 @@ func TestIndex(t *testing.T) { t.Fatal(err) } got, err := index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != nil { t.Fatal(err) @@ -78,7 +80,7 @@ func TestIndex(t *testing.T) { t.Run("overwrite", func(t *testing.T) { want := IndexItem{ - Hash: []byte("put-hash"), + Address: []byte("put-hash"), Data: []byte("New DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), } @@ -88,7 +90,7 @@ func TestIndex(t *testing.T) { t.Fatal(err) } got, err := index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != nil { t.Fatal(err) @@ -99,7 +101,7 @@ func TestIndex(t *testing.T) { t.Run("put in batch", func(t *testing.T) { want := IndexItem{ - Hash: []byte("put-in-batch-hash"), + Address: []byte("put-in-batch-hash"), Data: []byte("DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), } @@ -111,7 +113,7 @@ func TestIndex(t *testing.T) { t.Fatal(err) } got, err := index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != nil { t.Fatal(err) @@ -120,7 +122,7 @@ func TestIndex(t *testing.T) { t.Run("overwrite", func(t *testing.T) { want := IndexItem{ - Hash: []byte("put-in-batch-hash"), + Address: []byte("put-in-batch-hash"), Data: []byte("New DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), } @@ -132,7 +134,7 @@ func TestIndex(t *testing.T) { t.Fatal(err) } got, err := index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != nil { t.Fatal(err) @@ -143,7 +145,7 @@ func TestIndex(t *testing.T) { t.Run("delete", func(t *testing.T) { want := IndexItem{ - Hash: []byte("delete-hash"), + Address: []byte("delete-hash"), Data: []byte("DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), } @@ -153,7 +155,7 @@ func TestIndex(t *testing.T) { t.Fatal(err) } got, err := index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != nil { t.Fatal(err) @@ -161,14 +163,14 @@ func TestIndex(t *testing.T) { checkIndexItem(t, got, want) err = index.Delete(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != nil { t.Fatal(err) } got, err = index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != leveldb.ErrNotFound { t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound) @@ -177,7 +179,7 @@ func TestIndex(t *testing.T) { t.Run("delete in batch", func(t *testing.T) { want := IndexItem{ - Hash: []byte("delete-in-batch-hash"), + Address: []byte("delete-in-batch-hash"), Data: []byte("DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), } @@ -187,7 +189,7 @@ func TestIndex(t *testing.T) { t.Fatal(err) } got, err := index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != nil { t.Fatal(err) @@ -196,7 +198,7 @@ func TestIndex(t *testing.T) { batch := new(leveldb.Batch) index.DeleteInBatch(batch, IndexItem{ - Hash: want.Hash, + Address: want.Address, }) err = db.WriteBatch(batch) if err != nil { @@ -204,7 +206,7 @@ func TestIndex(t *testing.T) { } got, err = index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != leveldb.ErrNotFound { t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound) @@ -212,6 +214,7 @@ func TestIndex(t *testing.T) { }) } +// TestIndex_iterate validates index iterator functions for correctness. func TestIndex_iterate(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -223,24 +226,24 @@ func TestIndex_iterate(t *testing.T) { items := []IndexItem{ { - Hash: []byte("iterate-hash-01"), - Data: []byte("data80"), + Address: []byte("iterate-hash-01"), + Data: []byte("data80"), }, { - Hash: []byte("iterate-hash-03"), - Data: []byte("data22"), + Address: []byte("iterate-hash-03"), + Data: []byte("data22"), }, { - Hash: []byte("iterate-hash-05"), - Data: []byte("data41"), + Address: []byte("iterate-hash-05"), + Data: []byte("data41"), }, { - Hash: []byte("iterate-hash-02"), - Data: []byte("data84"), + Address: []byte("iterate-hash-02"), + Data: []byte("data84"), }, { - Hash: []byte("iterate-hash-06"), - Data: []byte("data1"), + Address: []byte("iterate-hash-06"), + Data: []byte("data1"), }, } batch := new(leveldb.Batch) @@ -252,8 +255,8 @@ func TestIndex_iterate(t *testing.T) { t.Fatal(err) } item04 := IndexItem{ - Hash: []byte("iterate-hash-04"), - Data: []byte("data0"), + Address: []byte("iterate-hash-04"), + Data: []byte("data0"), } err = index.Put(item04) if err != nil { @@ -262,7 +265,7 @@ func TestIndex_iterate(t *testing.T) { items = append(items, item04) sort.SliceStable(items, func(i, j int) bool { - return bytes.Compare(items[i].Hash, items[j].Hash) < 0 + return bytes.Compare(items[i].Address, items[j].Address) < 0 }) t.Run("all", func(t *testing.T) { @@ -331,8 +334,8 @@ func TestIndex_iterate(t *testing.T) { } secondIndexItem := IndexItem{ - Hash: []byte("iterate-hash-10"), - Data: []byte("data-second"), + Address: []byte("iterate-hash-10"), + Data: []byte("data-second"), } err = secondIndex.Put(secondIndexItem) if err != nil { @@ -368,11 +371,12 @@ func TestIndex_iterate(t *testing.T) { }) } +// checkIndexItem is a test helper function that compares if two Index items are the same. func checkIndexItem(t *testing.T, got, want IndexItem) { t.Helper() - if !bytes.Equal(got.Hash, want.Hash) { - t.Errorf("got hash %q, expected %q", string(got.Hash), string(want.Hash)) + if !bytes.Equal(got.Address, want.Address) { + t.Errorf("got hash %q, expected %q", string(got.Address), string(want.Address)) } if !bytes.Equal(got.Data, want.Data) { t.Errorf("got data %q, expected %q", string(got.Data), string(want.Data)) diff --git a/swarm/shed/internal/example_dbstore_test.go b/swarm/shed/internal/example_dbstore_test.go deleted file mode 100644 index a9fd0372719..00000000000 --- a/swarm/shed/internal/example_dbstore_test.go +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package internal_test - -import ( - "bytes" - "context" - "encoding/binary" - "fmt" - "io/ioutil" - "log" - "os" - "time" - - "github.com/ethereum/go-ethereum/swarm/shed/internal" - "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/syndtr/goleveldb/leveldb" -) - -// DB is just an example for composing indexes. -type DB struct { - db *internal.DB - - // fields and indexes - schemaName internal.StringField - sizeCounter internal.Uint64Field - accessCounter internal.Uint64Field - retrievalIndex internal.Index - accessIndex internal.Index - gcIndex internal.Index -} - -func New(path string) (db *DB, err error) { - idb, err := internal.NewDB(path) - if err != nil { - return nil, err - } - db = &DB{ - db: idb, - } - db.schemaName, err = idb.NewStringField("schema-name") - if err != nil { - return nil, err - } - db.sizeCounter, err = idb.NewUint64Field("size-counter") - if err != nil { - return nil, err - } - db.accessCounter, err = idb.NewUint64Field("access-counter") - if err != nil { - return nil, err - } - db.retrievalIndex, err = idb.NewIndex("Hash->StoreTimestamp|Data", internal.IndexFuncs{ - EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { - return fields.Hash, nil - }, - DecodeKey: func(key []byte) (e internal.IndexItem, err error) { - e.Hash = key - return e, nil - }, - EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { - b := make([]byte, 8) - binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) - value = append(b, fields.Data...) - return value, nil - }, - DecodeValue: func(value []byte) (e internal.IndexItem, err error) { - e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) - e.Data = value[8:] - return e, nil - }, - }) - db.accessIndex, err = idb.NewIndex("Hash->AccessTimestamp", internal.IndexFuncs{ - EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { - return fields.Hash, nil - }, - DecodeKey: func(key []byte) (e internal.IndexItem, err error) { - e.Hash = key - return e, nil - }, - EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { - b := make([]byte, 8) - binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp)) - return b, nil - }, - DecodeValue: func(value []byte) (e internal.IndexItem, err error) { - e.AccessTimestamp = int64(binary.BigEndian.Uint64(value)) - return e, nil - }, - }) - db.gcIndex, err = idb.NewIndex("AccessTimestamp|StoredTimestamp|Hash->nil", internal.IndexFuncs{ - EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { - b := make([]byte, 16, 16+len(fields.Hash)) - binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp)) - binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp)) - key = append(b, fields.Hash...) - return key, nil - }, - DecodeKey: func(key []byte) (e internal.IndexItem, err error) { - e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8])) - e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16])) - e.Hash = key[16:] - return e, nil - }, - EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { - return nil, nil - }, - DecodeValue: func(value []byte) (e internal.IndexItem, err error) { - return e, nil - }, - }) - if err != nil { - return nil, err - } - return db, nil -} - -func (db *DB) Put(_ context.Context, ch storage.Chunk) (err error) { - return db.retrievalIndex.Put(internal.IndexItem{ - Hash: ch.Address(), - Data: ch.Data(), - StoreTimestamp: time.Now().UTC().UnixNano(), - }) -} - -func (db *DB) Get(_ context.Context, ref storage.Address) (c storage.Chunk, err error) { - batch := new(leveldb.Batch) - - item, err := db.retrievalIndex.Get(internal.IndexItem{ - Hash: ref, - }) - if err != nil { - if err == leveldb.ErrNotFound { - return nil, storage.ErrChunkNotFound - } - return nil, err - } - - accessItem, err := db.accessIndex.Get(internal.IndexItem{ - Hash: ref, - }) - switch err { - case nil: - err = db.gcIndex.DeleteInBatch(batch, internal.IndexItem{ - Hash: item.Hash, - StoreTimestamp: accessItem.AccessTimestamp, - AccessTimestamp: item.StoreTimestamp, - }) - if err != nil { - return nil, err - } - case leveldb.ErrNotFound: - default: - return nil, err - } - - accessTimestamp := time.Now().UTC().UnixNano() - - err = db.accessIndex.PutInBatch(batch, internal.IndexItem{ - Hash: ref, - AccessTimestamp: accessTimestamp, - }) - if err != nil { - return nil, err - } - - err = db.gcIndex.PutInBatch(batch, internal.IndexItem{ - Hash: item.Hash, - AccessTimestamp: accessTimestamp, - StoreTimestamp: item.StoreTimestamp, - }) - if err != nil { - return nil, err - } - - err = db.db.WriteBatch(batch) - if err != nil { - return nil, err - } - - return storage.NewChunk(item.Hash, item.Data), nil -} - -func (db *DB) CollectGarbage() (err error) { - const maxTrashSize = 100 - maxRounds := 10 // adbitrary number, needs to be calculated - - for roundCount := 0; roundCount < maxRounds; roundCount++ { - var garbageCount int - trash := new(leveldb.Batch) - err = db.gcIndex.IterateAll(func(item internal.IndexItem) (stop bool, err error) { - err = db.retrievalIndex.DeleteInBatch(trash, item) - if err != nil { - return false, err - } - err = db.accessIndex.DeleteInBatch(trash, item) - if err != nil { - return false, err - } - err = db.gcIndex.DeleteInBatch(trash, item) - if err != nil { - return false, err - } - garbageCount++ - if garbageCount >= maxTrashSize { - return true, nil - } - return false, nil - }) - if err != nil { - return err - } - if garbageCount == 0 { - return nil - } - err = db.db.WriteBatch(trash) - if err != nil { - return err - } - } - return nil -} - -func (db *DB) GetSchema() (name string, err error) { - name, err = db.schemaName.Get() - if err == leveldb.ErrNotFound { - return "", nil - } - return name, err -} - -func (db *DB) PutSchema(name string) (err error) { - return db.schemaName.Put(name) -} - -func (db *DB) Close() { - db.db.Close() -} - -func Example_dbstore() { - dir, err := ioutil.TempDir("", "ephemeral") - if err != nil { - log.Fatal(err) - } - defer os.RemoveAll(dir) - - db, err := New(dir) - if err != nil { - log.Fatal(err) - } - - ch := storage.GenerateRandomChunk(1024) - err = db.Put(context.Background(), ch) - if err != nil { - log.Fatal(err) - } - - got, err := db.Get(context.Background(), ch.Address()) - if err != nil { - log.Fatal(err) - } - - fmt.Println(bytes.Equal(got.Data(), ch.Data())) - - //Output: true -} diff --git a/swarm/shed/internal/schema.go b/swarm/shed/schema.go similarity index 63% rename from swarm/shed/internal/schema.go rename to swarm/shed/schema.go index 9440ba89558..579d058aba1 100644 --- a/swarm/shed/internal/schema.go +++ b/swarm/shed/schema.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "encoding/json" @@ -23,24 +23,37 @@ import ( ) var ( - keySchema = []byte{0} - keyPrefixFields byte = 1 - keyPrefixIndexStart byte = 2 // Q: or maybe 7, to have more space for potential specific perfixes + // LevelDB key value for storing the schema. + keySchema = []byte{0} + // LevelDB key prefix for all field type. + // LevelDB keys will be constructed by appending name values to this prefix. + keyPrefixFields byte = 1 + // LevelDB key prefix from which indexing keys start. + // Every index has its own key prefix and this value defines the first one. + keyPrefixIndexStart byte = 2 // Q: or maybe a higher number like 7, to have more space for potential specific perfixes ) +// schema is used to serialize known database structure information. type schema struct { - Fields map[string]fieldSpec `json:"fields"` - Indexes map[byte]indexSpec `json:"indexes"` + Fields map[string]fieldSpec `json:"fields"` // keys are field names + Indexes map[byte]indexSpec `json:"indexes"` // keys are index prefix bytes } +// fieldSpec holds information about a particular field. +// It does not need Name field as it is contained in the +// schema.Field map key. type fieldSpec struct { Type string `json:"type"` } +// indxSpec holds information about a particular index. +// It does not contain index type, as indexes do not have type. type indexSpec struct { Name string `json:"name"` } +// schemaFieldKey retrives the complete LevelDB key for +// a particular field form the schema definition. func (db *DB) schemaFieldKey(name, fieldType string) (key []byte, err error) { if name == "" { return nil, errors.New("filed name can not be blank") @@ -73,7 +86,9 @@ func (db *DB) schemaFieldKey(name, fieldType string) (key []byte, err error) { return append([]byte{keyPrefixFields}, []byte(name)...), nil } -func (db *DB) schemaIndexID(name string) (id byte, err error) { +// schemaIndexID retrieves the complete LevelDB prefix for +// a particular index. +func (db *DB) schemaIndexPrefix(name string) (id byte, err error) { if name == "" { return 0, errors.New("index name can not be blank") } @@ -97,6 +112,8 @@ func (db *DB) schemaIndexID(name string) (id byte, err error) { return id, db.putSchema(s) } +// getSchema retrieves the complete schema from +// the database. func (db *DB) getSchema() (s schema, err error) { b, err := db.Get(keySchema) if err != nil { @@ -106,6 +123,8 @@ func (db *DB) getSchema() (s schema, err error) { return s, err } +// putSchema stores the complete schema to +// the database. func (db *DB) putSchema(s schema) (err error) { b, err := json.Marshal(s) if err != nil { diff --git a/swarm/shed/internal/schema_test.go b/swarm/shed/schema_test.go similarity index 87% rename from swarm/shed/internal/schema_test.go rename to swarm/shed/schema_test.go index 8b3682a8e95..a0c1838c8bc 100644 --- a/swarm/shed/internal/schema_test.go +++ b/swarm/shed/schema_test.go @@ -14,14 +14,15 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "bytes" "testing" ) -func TestSchema_schemaFieldKey(t *testing.T) { +// TestDB_schemaFieldKey validates correctness of schemaFieldKey. +func TestDB_schemaFieldKey(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -86,17 +87,18 @@ func TestSchema_schemaFieldKey(t *testing.T) { }) } -func TestSchema_schemaIndexID(t *testing.T) { +// TestDB_schemaIndexPrefix validates correctness of schemaIndexPrefix. +func TestDB_schemaIndexPrefix(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() t.Run("same name", func(t *testing.T) { - id1, err := db.schemaIndexID("test") + id1, err := db.schemaIndexPrefix("test") if err != nil { t.Fatal(err) } - id2, err := db.schemaIndexID("test") + id2, err := db.schemaIndexPrefix("test") if err != nil { t.Fatal(err) } @@ -107,12 +109,12 @@ func TestSchema_schemaIndexID(t *testing.T) { }) t.Run("different names", func(t *testing.T) { - id1, err := db.schemaIndexID("test1") + id1, err := db.schemaIndexPrefix("test1") if err != nil { t.Fatal(err) } - id2, err := db.schemaIndexID("test2") + id2, err := db.schemaIndexPrefix("test2") if err != nil { t.Fatal(err) } diff --git a/swarm/shed/shed.go b/swarm/shed/shed.go deleted file mode 100644 index 4f65d4ed814..00000000000 --- a/swarm/shed/shed.go +++ /dev/null @@ -1,247 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package shed - -import ( - "context" - "encoding/binary" - "time" - - "github.com/ethereum/go-ethereum/swarm/shed/internal" - "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/syndtr/goleveldb/leveldb" -) - -// DB is just an example for composing indexes. -type DB struct { - db *internal.DB - - // fields and indexes - schemaName internal.StringField - sizeCounter internal.Uint64Field - accessCounter internal.Uint64Field - retrievalIndex internal.Index - accessIndex internal.Index - gcIndex internal.Index -} - -func New(path string) (db *DB, err error) { - idb, err := internal.NewDB(path) - if err != nil { - return nil, err - } - db = &DB{ - db: idb, - } - db.schemaName, err = idb.NewStringField("schema-name") - if err != nil { - return nil, err - } - db.sizeCounter, err = idb.NewUint64Field("size-counter") - if err != nil { - return nil, err - } - db.accessCounter, err = idb.NewUint64Field("access-counter") - if err != nil { - return nil, err - } - db.retrievalIndex, err = idb.NewIndex("Hash->StoreTimestamp|Data", internal.IndexFuncs{ - EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { - return fields.Hash, nil - }, - DecodeKey: func(key []byte) (e internal.IndexItem, err error) { - e.Hash = key - return e, nil - }, - EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { - b := make([]byte, 8) - binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) - value = append(b, fields.Data...) - return value, nil - }, - DecodeValue: func(value []byte) (e internal.IndexItem, err error) { - e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) - e.Data = value[8:] - return e, nil - }, - }) - db.accessIndex, err = idb.NewIndex("Hash->AccessTimestamp", internal.IndexFuncs{ - EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { - return fields.Hash, nil - }, - DecodeKey: func(key []byte) (e internal.IndexItem, err error) { - e.Hash = key - return e, nil - }, - EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { - b := make([]byte, 8) - binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp)) - return b, nil - }, - DecodeValue: func(value []byte) (e internal.IndexItem, err error) { - e.AccessTimestamp = int64(binary.BigEndian.Uint64(value)) - return e, nil - }, - }) - db.gcIndex, err = idb.NewIndex("AccessTimestamp|StoredTimestamp|Hash->nil", internal.IndexFuncs{ - EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { - b := make([]byte, 16, 16+len(fields.Hash)) - binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp)) - binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp)) - key = append(b, fields.Hash...) - return key, nil - }, - DecodeKey: func(key []byte) (e internal.IndexItem, err error) { - e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8])) - e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16])) - e.Hash = key[16:] - return e, nil - }, - EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { - return nil, nil - }, - DecodeValue: func(value []byte) (e internal.IndexItem, err error) { - return e, nil - }, - }) - if err != nil { - return nil, err - } - return db, nil -} - -func (db *DB) Put(_ context.Context, ch storage.Chunk) (err error) { - return db.retrievalIndex.Put(internal.IndexItem{ - Hash: ch.Address(), - Data: ch.Data(), - StoreTimestamp: time.Now().UTC().UnixNano(), - }) -} - -func (db *DB) Get(_ context.Context, ref storage.Address) (c storage.Chunk, err error) { - batch := new(leveldb.Batch) - - item, err := db.retrievalIndex.Get(internal.IndexItem{ - Hash: ref, - }) - if err != nil { - if err == leveldb.ErrNotFound { - return nil, storage.ErrChunkNotFound - } - return nil, err - } - - accessItem, err := db.accessIndex.Get(internal.IndexItem{ - Hash: ref, - }) - switch err { - case nil: - err = db.gcIndex.DeleteInBatch(batch, internal.IndexItem{ - Hash: item.Hash, - StoreTimestamp: accessItem.AccessTimestamp, - AccessTimestamp: item.StoreTimestamp, - }) - if err != nil { - return nil, err - } - case leveldb.ErrNotFound: - default: - return nil, err - } - - accessTimestamp := time.Now().UTC().UnixNano() - - err = db.accessIndex.PutInBatch(batch, internal.IndexItem{ - Hash: ref, - AccessTimestamp: accessTimestamp, - }) - if err != nil { - return nil, err - } - - err = db.gcIndex.PutInBatch(batch, internal.IndexItem{ - Hash: item.Hash, - AccessTimestamp: accessTimestamp, - StoreTimestamp: item.StoreTimestamp, - }) - if err != nil { - return nil, err - } - - err = db.db.WriteBatch(batch) - if err != nil { - return nil, err - } - - return storage.NewChunk(item.Hash, item.Data), nil -} - -func (db *DB) CollectGarbage() (err error) { - const maxTrashSize = 100 - maxRounds := 10 // adbitrary number, needs to be calculated - - for roundCount := 0; roundCount < maxRounds; roundCount++ { - var garbageCount int - trash := new(leveldb.Batch) - err = db.gcIndex.IterateAll(func(item internal.IndexItem) (stop bool, err error) { - err = db.retrievalIndex.DeleteInBatch(trash, item) - if err != nil { - return false, err - } - err = db.accessIndex.DeleteInBatch(trash, item) - if err != nil { - return false, err - } - err = db.gcIndex.DeleteInBatch(trash, item) - if err != nil { - return false, err - } - garbageCount++ - if garbageCount >= maxTrashSize { - return true, nil - } - return false, nil - }) - if err != nil { - return err - } - if garbageCount == 0 { - return nil - } - err = db.db.WriteBatch(trash) - if err != nil { - return err - } - } - return nil -} - -func (db *DB) GetSchema() (name string, err error) { - name, err = db.schemaName.Get() - if err == leveldb.ErrNotFound { - return "", nil - } - return name, err -} - -func (db *DB) PutSchema(name string) (err error) { - return db.schemaName.Put(name) -} - -func (db *DB) Close() { - db.db.Close() -} From ba8d0f0e505d9ad717c119af4957cd5f89dbe940 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Thu, 15 Nov 2018 17:24:13 +0100 Subject: [PATCH 04/16] swarm/shed: replace JSONField with StructField and rlp encoding --- swarm/shed/{field_json.go => field_struct.go} | 29 +++++++++---------- ...ield_json_test.go => field_struct_test.go} | 8 ++--- 2 files changed, 18 insertions(+), 19 deletions(-) rename swarm/shed/{field_json.go => field_struct.go} (69%) rename swarm/shed/{field_json_test.go => field_struct_test.go} (94%) diff --git a/swarm/shed/field_json.go b/swarm/shed/field_struct.go similarity index 69% rename from swarm/shed/field_json.go rename to swarm/shed/field_struct.go index b3274bb1b65..93d6e940964 100644 --- a/swarm/shed/field_json.go +++ b/swarm/shed/field_struct.go @@ -17,26 +17,25 @@ package shed import ( - "encoding/json" - + "github.com/ethereum/go-ethereum/rlp" "github.com/syndtr/goleveldb/leveldb" ) -// JSONField is a helper to store complex structure by -// encoding it in JSON format. -type JSONField struct { +// StructField is a helper to store complex structure by +// encoding it in RLP format. +type StructField struct { db *DB key []byte } -// NewJSONField returns a new JSONField. +// NewStructField returns a new StructField. // It validates its name and type against the database schema. -func (db *DB) NewJSONField(name string) (f JSONField, err error) { - key, err := db.schemaFieldKey(name, "json") +func (db *DB) NewStructField(name string) (f StructField, err error) { + key, err := db.schemaFieldKey(name, "struct-rlp") if err != nil { return f, err } - return JSONField{ + return StructField{ db: db, key: key, }, nil @@ -44,17 +43,17 @@ func (db *DB) NewJSONField(name string) (f JSONField, err error) { // Unmarshal unmarshals data fromt he database to a provided val. // If the data is not found leveldb.ErrNotFound is returned. -func (f JSONField) Unmarshal(val interface{}) (err error) { +func (f StructField) Unmarshal(val interface{}) (err error) { b, err := f.db.Get(f.key) if err != nil { return err } - return json.Unmarshal(b, val) + return rlp.DecodeBytes(b, val) } // Put marshals provided val and saves it to the database. -func (f JSONField) Put(val interface{}) (err error) { - b, err := json.Marshal(val) +func (f StructField) Put(val interface{}) (err error) { + b, err := rlp.EncodeToBytes(val) if err != nil { return err } @@ -62,8 +61,8 @@ func (f JSONField) Put(val interface{}) (err error) { } // PutInBatch marshals provided val and puts it into the batch. -func (f JSONField) PutInBatch(batch *leveldb.Batch, val interface{}) (err error) { - b, err := json.Marshal(val) +func (f StructField) PutInBatch(batch *leveldb.Batch, val interface{}) (err error) { + b, err := rlp.EncodeToBytes(val) if err != nil { return err } diff --git a/swarm/shed/field_json_test.go b/swarm/shed/field_struct_test.go similarity index 94% rename from swarm/shed/field_json_test.go rename to swarm/shed/field_struct_test.go index c9e2cf658d9..137ac3a0eda 100644 --- a/swarm/shed/field_json_test.go +++ b/swarm/shed/field_struct_test.go @@ -22,13 +22,13 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) -// TestJSONField validates put and unmarshal operations -// of the JSONField. -func TestJSONField(t *testing.T) { +// TestStructField validates put and unmarshal operations +// of the StructField. +func TestStructField(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() - complexField, err := db.NewJSONField("complex-field") + complexField, err := db.NewStructField("complex-field") if err != nil { t.Fatal(err) } From 4696fb29c361427d61c4018b4c45a0814023bb36 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Mon, 19 Nov 2018 14:32:07 +0100 Subject: [PATCH 05/16] swarm/shed, swarm/storage/mock: add MockIndex to shed --- swarm/shed/example_store_test.go | 36 ++++++++-- swarm/shed/index.go | 83 +++++++++++++++++++---- swarm/shed/index_mock.go | 113 +++++++++++++++++++++++++++++++ swarm/shed/index_mock_test.go | 45 ++++++++++++ swarm/shed/index_test.go | 52 +++++++++++--- swarm/storage/mock/db/db.go | 7 ++ swarm/storage/mock/mem/mem.go | 16 +++++ swarm/storage/mock/mock.go | 7 ++ swarm/storage/mock/rpc/rpc.go | 6 ++ swarm/storage/mock/test/test.go | 53 +++++++++++++++ 10 files changed, 393 insertions(+), 25 deletions(-) create mode 100644 swarm/shed/index_mock.go create mode 100644 swarm/shed/index_mock_test.go diff --git a/swarm/shed/example_store_test.go b/swarm/shed/example_store_test.go index 733a972c65e..d1e6aa463b5 100644 --- a/swarm/shed/example_store_test.go +++ b/swarm/shed/example_store_test.go @@ -24,10 +24,14 @@ import ( "io/ioutil" "log" "os" + "strings" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/swarm/shed" "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" "github.com/syndtr/goleveldb/leveldb" ) @@ -43,7 +47,7 @@ type Store struct { schemaName shed.StringField sizeCounter shed.Uint64Field accessCounter shed.Uint64Field - retrievalIndex shed.Index + retrievalIndex shed.IndexInterface // example of swapable Index/MockIndex accessIndex shed.Index gcIndex shed.Index } @@ -51,7 +55,7 @@ type Store struct { // New returns new Store. All fields and indexes are initialized // and possible conflicts with schema from existing database is checked // automatically. -func New(path string) (s *Store, err error) { +func New(path string, mockStore *mock.NodeStore) (s *Store, err error) { db, err := shed.NewDB(path) if err != nil { return nil, err @@ -70,7 +74,7 @@ func New(path string) (s *Store, err error) { return nil, err } // Index storing actual chunk address, data and store timestamp. - s.retrievalIndex, err = db.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{ + retrievalIndex, err := db.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{ EncodeKey: func(fields shed.IndexItem) (key []byte, err error) { return fields.Address, nil }, @@ -90,6 +94,14 @@ func New(path string) (s *Store, err error) { return e, nil }, }) + if err != nil { + return nil, err + } + s.retrievalIndex = retrievalIndex + if mockStore != nil { + // If mock store is provided, use it for retrieval index. + s.retrievalIndex = retrievalIndex.NewMockIndex(mockStore) + } // Index storing access timestamp for a particular address. // It is needed in order to update gc index keys for iteration order. s.accessIndex, err = db.NewIndex("Address->AccessTimestamp", shed.IndexFuncs{ @@ -110,6 +122,9 @@ func New(path string) (s *Store, err error) { return e, nil }, }) + if err != nil { + return nil, err + } // Index with keys ordered by access timestamp for garbage collection prioritization. s.gcIndex, err = db.NewIndex("AccessTimestamp|StoredTimestamp|Address->nil", shed.IndexFuncs{ EncodeKey: func(fields shed.IndexItem) (key []byte, err error) { @@ -303,7 +318,20 @@ func Example_store() { } defer os.RemoveAll(dir) - s, err := New(dir) + var mockStore *mock.NodeStore + // Configuring mock store using environment variable is + // just an example, it can be accomplished in more elegant ways. + if strings.EqualFold(os.Getenv("SWARM_USE_MOCKSTORE"), "true") { + // Global store is constructed here for example purposes. + // It should be available globally so that all nodes can access it. + globalStore := mem.NewGlobalStore() + // An arbitrary address is used for this example. + // In real situations this address should be the same as the Swarm + // node address. + mockStore = globalStore.NewNodeStore(common.HexToAddress("12345678")) + } + + s, err := New(dir, mockStore) if err != nil { log.Fatal(err) } diff --git a/swarm/shed/index.go b/swarm/shed/index.go index d1cf7a757f2..04ed0f19c97 100644 --- a/swarm/shed/index.go +++ b/swarm/shed/index.go @@ -17,6 +17,7 @@ package shed import ( + "github.com/ethereum/go-ethereum/swarm/storage/mock" "github.com/syndtr/goleveldb/leveldb" ) @@ -64,6 +65,7 @@ func (i IndexItem) Join(i2 IndexItem) (new IndexItem) { // - getting a particular IndexItem // - saving a particular IndexItem // - iterating over a sorted LevelDB keys +// It implements IndexIteratorInterface interface. type Index struct { db *DB prefix []byte @@ -92,18 +94,10 @@ func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) { } prefix := []byte{id} return Index{ - db: db, - prefix: prefix, - encodeKeyFunc: func(e IndexItem) (key []byte, err error) { - key, err = funcs.EncodeKey(e) - if err != nil { - return nil, err - } - return append(append(make([]byte, 0, len(key)+1), prefix...), key...), nil - }, - decodeKeyFunc: func(key []byte) (e IndexItem, err error) { - return funcs.DecodeKey(key[1:]) - }, + db: db, + prefix: prefix, + encodeKeyFunc: newIndexEncodeKeyFunc(funcs.EncodeKey, id), + decodeKeyFunc: newDecodeKeyFunc(funcs.DecodeKey), encodeValueFunc: funcs.EncodeValue, decodeValueFunc: funcs.DecodeValue, }, nil @@ -248,3 +242,68 @@ func (f Index) IterateFrom(start IndexItem, fn IndexIterFunc) (err error) { } return it.Error() } + +// NewMockIndex is a helper function to easily construct MockIndex +// when from the same definition of Index. +func (f Index) NewMockIndex(store *mock.NodeStore) (m MockIndex) { + return MockIndex{ + store: store, + prefix: f.prefix, + encodeKeyFunc: f.encodeKeyFunc, + decodeKeyFunc: f.decodeKeyFunc, + encodeValueFunc: f.encodeValueFunc, + decodeValueFunc: f.decodeValueFunc, + } +} + +// IndexInterface defines methods that are required for a simple index +// that does not use iterations. +// It can be used when Index should be swapped with MockIndex or any other +// IndexInterface implementation. +// In most cases, interface is not needed and it is recommended to use the +// Index implementation whenever possible. +type IndexInterface interface { + Get(keyFields IndexItem) (out IndexItem, err error) + Put(i IndexItem) (err error) + PutInBatch(batch *leveldb.Batch, i IndexItem) (err error) + Delete(keyFields IndexItem) (err error) + DeleteInBatch(batch *leveldb.Batch, keyFields IndexItem) (err error) +} + +// IndexIteratorInterface defines metods for a full index implementation with +// iterators. Index type implements this type. +// In most cases, interface is not needed and it is recommended to use the +// Index implementation whenever possible. +type IndexIteratorInterface interface { + IndexInterface + IterateAll(fn IndexIterFunc) (err error) + IterateFrom(start IndexItem, fn IndexIterFunc) (err error) +} + +// newIndexEncodeKeyFunc adjusts Index and MockIndex LevelDB key +// by appending the provided index id byte. +// This is needed to avoid collisions between keys of different +// indexes as all index ids are unique. +func newIndexEncodeKeyFunc( + encodeKeyFunc func(fields IndexItem) (key []byte, err error), + id byte, +) (f func(e IndexItem) (key []byte, err error)) { + prefix := []byte{id} + return func(e IndexItem) (key []byte, err error) { + key, err = encodeKeyFunc(e) + if err != nil { + return nil, err + } + return append(append(make([]byte, 0, len(key)+1), prefix...), key...), nil + } +} + +// newDecodeKeyFunc reverses the newIndexEncodeKeyFunc constructd key +// to transparently work with index keys without their index ids. +// This function is used in NewIndex and NewMockIndex constructors. +// It assumes that index keys are prefixed with only one byte. +func newDecodeKeyFunc(decodeKeyFunc func(key []byte) (e IndexItem, err error)) (f func(key []byte) (e IndexItem, err error)) { + return func(key []byte) (e IndexItem, err error) { + return decodeKeyFunc(key[1:]) + } +} diff --git a/swarm/shed/index_mock.go b/swarm/shed/index_mock.go new file mode 100644 index 00000000000..d9b7187a3dc --- /dev/null +++ b/swarm/shed/index_mock.go @@ -0,0 +1,113 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "github.com/ethereum/go-ethereum/swarm/storage/mock" + "github.com/syndtr/goleveldb/leveldb" +) + +// MockIndex provides a way to inject a mock.NodeStore to store +// data centrally instead on provided DB. DB is used just for schema +// validation and identifying byte prefix for the particular index +// that is mocked. +// Iterator functions are not implemented and MockIndex can not replace +// indexes that rely on them. +// It implements IndexField interface. +type MockIndex struct { + store *mock.NodeStore + prefix []byte + encodeKeyFunc func(fields IndexItem) (key []byte, err error) + decodeKeyFunc func(key []byte) (e IndexItem, err error) + encodeValueFunc func(fields IndexItem) (value []byte, err error) + decodeValueFunc func(value []byte) (e IndexItem, err error) +} + +// NewMockIndex returns a new MockIndex instance with defined name and +// encoding functions. The name must be unique and will be validated +// on database schema for a key prefix byte. +// The data will not be saved on the DB itself, but using a provided +// mock.NodeStore. +func (db *DB) NewMockIndex(store *mock.NodeStore, name string, funcs IndexFuncs) (f MockIndex, err error) { + id, err := db.schemaIndexPrefix(name) + if err != nil { + return f, err + } + return MockIndex{ + store: store, + prefix: []byte{id}, + encodeKeyFunc: newIndexEncodeKeyFunc(funcs.EncodeKey, id), + decodeKeyFunc: newDecodeKeyFunc(funcs.DecodeKey), + encodeValueFunc: funcs.EncodeValue, + decodeValueFunc: funcs.DecodeValue, + }, nil +} + +// Get accepts key fields represented as IndexItem to retrieve a +// value from the index and return maximum available information +// from the index represented as another IndexItem. +func (f MockIndex) Get(keyFields IndexItem) (out IndexItem, err error) { + key, err := f.encodeKeyFunc(keyFields) + if err != nil { + return out, err + } + value, err := f.store.Get(key) + if err != nil { + return out, err + } + out, err = f.decodeValueFunc(value) + if err != nil { + return out, err + } + return out.Join(keyFields), nil +} + +// Put accepts IndexItem to encode information from it +// and save it to the database. +func (f MockIndex) Put(i IndexItem) (err error) { + key, err := f.encodeKeyFunc(i) + if err != nil { + return err + } + value, err := f.encodeValueFunc(i) + if err != nil { + return err + } + return f.store.Put(key, value) +} + +// PutInBatch is the same as Put method. +// Batch is ignored and the data is saved to the mock store instantly. +func (f MockIndex) PutInBatch(_ *leveldb.Batch, i IndexItem) (err error) { + return f.Put(i) +} + +// Delete accepts IndexItem to remove a key/value pair +// form the database based on its fields. +func (f MockIndex) Delete(keyFields IndexItem) (err error) { + key, err := f.encodeKeyFunc(keyFields) + if err != nil { + return err + } + return f.store.Delete(key) +} + +// DeleteInBatch is the same as Delete just the operation. +// Batch is ignored and the data is deleted on the mock store instantly. +func (f MockIndex) DeleteInBatch(_ *leveldb.Batch, keyFields IndexItem) (err error) { + return f.Delete(keyFields) +} diff --git a/swarm/shed/index_mock_test.go b/swarm/shed/index_mock_test.go new file mode 100644 index 00000000000..adc9dd068fc --- /dev/null +++ b/swarm/shed/index_mock_test.go @@ -0,0 +1,45 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" +) + +// TestMockIndex validates put, get and delete functions of +// the MockIndex implementation. +func TestMockIndex(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + globalStore := mem.NewGlobalStore() + + index, err := db.NewMockIndex( + globalStore.NewNodeStore(common.HexToAddress("12345678")), + "retrieval", + retrievalIndexFuncs, + ) + if err != nil { + t.Fatal(err) + } + + testIndex(t, db, index) +} diff --git a/swarm/shed/index_test.go b/swarm/shed/index_test.go index 4e3eba326a8..a5c81091cc1 100644 --- a/swarm/shed/index_test.go +++ b/swarm/shed/index_test.go @@ -24,6 +24,9 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" "github.com/syndtr/goleveldb/leveldb" ) @@ -49,7 +52,7 @@ var retrievalIndexFuncs = IndexFuncs{ }, } -// TestIndex validates put, get and delete functions of the index. +// TestIndex validates put, get and delete functions of the Index implementation. func TestIndex(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -59,6 +62,29 @@ func TestIndex(t *testing.T) { t.Fatal(err) } + testIndex(t, db, index) +} + +// TestIndex_NewMockIndex validates put, get and delete functions of +// the MockIndex implementation, when constructed with Index.NewMockIndex function. +func TestIndex_NewMockIndex(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + index, err := db.NewIndex("retrieval", retrievalIndexFuncs) + if err != nil { + t.Fatal(err) + } + + globalStore := mem.NewGlobalStore() + + mockIndex := index.NewMockIndex(globalStore.NewNodeStore(common.HexToAddress("12345678"))) + + testIndex(t, db, mockIndex) +} + +// testIndex validates put, get and delete functions of a index interface. +func testIndex(t *testing.T, db *DB, index IndexInterface) { t.Run("put", func(t *testing.T) { want := IndexItem{ Address: []byte("put-hash"), @@ -66,7 +92,7 @@ func TestIndex(t *testing.T) { StoreTimestamp: time.Now().UTC().UnixNano(), } - err = index.Put(want) + err := index.Put(want) if err != nil { t.Fatal(err) } @@ -108,7 +134,7 @@ func TestIndex(t *testing.T) { batch := new(leveldb.Batch) index.PutInBatch(batch, want) - db.WriteBatch(batch) + err := db.WriteBatch(batch) if err != nil { t.Fatal(err) } @@ -150,7 +176,7 @@ func TestIndex(t *testing.T) { StoreTimestamp: time.Now().UTC().UnixNano(), } - err = index.Put(want) + err := index.Put(want) if err != nil { t.Fatal(err) } @@ -169,11 +195,15 @@ func TestIndex(t *testing.T) { t.Fatal(err) } + wantErr := leveldb.ErrNotFound + if _, ok := index.(MockIndex); ok { + wantErr = mock.ErrNotFound + } got, err = index.Get(IndexItem{ Address: want.Address, }) - if err != leveldb.ErrNotFound { - t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound) + if err != wantErr { + t.Fatalf("got error %v, want %v", err, wantErr) } }) @@ -184,7 +214,7 @@ func TestIndex(t *testing.T) { StoreTimestamp: time.Now().UTC().UnixNano(), } - err = index.Put(want) + err := index.Put(want) if err != nil { t.Fatal(err) } @@ -205,11 +235,15 @@ func TestIndex(t *testing.T) { t.Fatal(err) } + wantErr := leveldb.ErrNotFound + if _, ok := index.(MockIndex); ok { + wantErr = mock.ErrNotFound + } got, err = index.Get(IndexItem{ Address: want.Address, }) - if err != leveldb.ErrNotFound { - t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound) + if err != wantErr { + t.Fatalf("got error %v, want %v", err, wantErr) } }) } diff --git a/swarm/storage/mock/db/db.go b/swarm/storage/mock/db/db.go index 43bfa24f05c..73ae199e8b0 100644 --- a/swarm/storage/mock/db/db.go +++ b/swarm/storage/mock/db/db.go @@ -86,6 +86,13 @@ func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error { return s.db.Write(batch, nil) } +// Delete removes the chunk reference to node with address addr. +func (s *GlobalStore) Delete(addr common.Address, key []byte) error { + batch := new(leveldb.Batch) + batch.Delete(nodeDBKey(addr, key)) + return s.db.Write(batch, nil) +} + // HasKey returns whether a node with addr contains the key. func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool { has, err := s.db.Has(nodeDBKey(addr, key), nil) diff --git a/swarm/storage/mock/mem/mem.go b/swarm/storage/mock/mem/mem.go index 8878309d0e7..3a0a2beb8d5 100644 --- a/swarm/storage/mock/mem/mem.go +++ b/swarm/storage/mock/mem/mem.go @@ -83,6 +83,22 @@ func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error { return nil } +// Delete removes the chunk data for node with address addr. +func (s *GlobalStore) Delete(addr common.Address, key []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + + var count int + if _, ok := s.nodes[string(key)]; ok { + delete(s.nodes[string(key)], addr) + count = len(s.nodes[string(key)]) + } + if count == 0 { + delete(s.data, string(key)) + } + return nil +} + // HasKey returns whether a node with addr contains the key. func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool { s.mu.Lock() diff --git a/swarm/storage/mock/mock.go b/swarm/storage/mock/mock.go index 81340f92747..1fb71b70a24 100644 --- a/swarm/storage/mock/mock.go +++ b/swarm/storage/mock/mock.go @@ -70,6 +70,12 @@ func (n *NodeStore) Put(key []byte, data []byte) error { return n.store.Put(n.addr, key, data) } +// Delete removes chunk data for a key for a node that has the address +// provided on NodeStore initialization. +func (n *NodeStore) Delete(key []byte) error { + return n.store.Delete(n.addr, key) +} + // GlobalStorer defines methods for mock db store // that stores chunk data for all swarm nodes. // It is used in tests to construct mock NodeStores @@ -77,6 +83,7 @@ func (n *NodeStore) Put(key []byte, data []byte) error { type GlobalStorer interface { Get(addr common.Address, key []byte) (data []byte, err error) Put(addr common.Address, key []byte, data []byte) error + Delete(addr common.Address, key []byte) error HasKey(addr common.Address, key []byte) bool // NewNodeStore creates an instance of NodeStore // to be used by a single swarm node with diff --git a/swarm/storage/mock/rpc/rpc.go b/swarm/storage/mock/rpc/rpc.go index 6e735f69884..8cd6c83a7a4 100644 --- a/swarm/storage/mock/rpc/rpc.go +++ b/swarm/storage/mock/rpc/rpc.go @@ -73,6 +73,12 @@ func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error { return err } +// Delete calls a Delete method to RPC server. +func (s *GlobalStore) Delete(addr common.Address, key []byte) error { + err := s.client.Call(nil, "mockStore_delete", addr, key) + return err +} + // HasKey calls a HasKey method to RPC server. func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool { var has bool diff --git a/swarm/storage/mock/test/test.go b/swarm/storage/mock/test/test.go index 02da3af553f..10180985f33 100644 --- a/swarm/storage/mock/test/test.go +++ b/swarm/storage/mock/test/test.go @@ -72,6 +72,31 @@ func MockStore(t *testing.T, globalStore mock.GlobalStorer, n int) { } } } + t.Run("delete", func(t *testing.T) { + chunkAddr := storage.Address([]byte("1234567890abcd")) + for _, addr := range addrs { + err := globalStore.Put(addr, chunkAddr, []byte("data")) + if err != nil { + t.Fatalf("put data to store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err) + } + } + firstNodeAddr := addrs[0] + if err := globalStore.Delete(firstNodeAddr, chunkAddr); err != nil { + t.Fatalf("delete from store %s key %s: %v", firstNodeAddr.Hex(), chunkAddr.Hex(), err) + } + for i, addr := range addrs { + _, err := globalStore.Get(addr, chunkAddr) + if i == 0 { + if err != mock.ErrNotFound { + t.Errorf("get data from store %s key %s: expected mock.ErrNotFound error, got %v", addr.Hex(), chunkAddr.Hex(), err) + } + } else { + if err != nil { + t.Errorf("get data from store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err) + } + } + } + }) }) t.Run("NodeStore", func(t *testing.T) { @@ -114,6 +139,34 @@ func MockStore(t *testing.T, globalStore mock.GlobalStorer, n int) { } } } + t.Run("delete", func(t *testing.T) { + chunkAddr := storage.Address([]byte("1234567890abcd")) + var chosenStore *mock.NodeStore + for addr, store := range nodes { + if chosenStore == nil { + chosenStore = store + } + err := store.Put(chunkAddr, []byte("data")) + if err != nil { + t.Fatalf("put data to store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err) + } + } + if err := chosenStore.Delete(chunkAddr); err != nil { + t.Fatalf("delete key %s: %v", chunkAddr.Hex(), err) + } + for addr, store := range nodes { + _, err := store.Get(chunkAddr) + if store == chosenStore { + if err != mock.ErrNotFound { + t.Errorf("get data from store %s key %s: expected mock.ErrNotFound error, got %v", addr.Hex(), chunkAddr.Hex(), err) + } + } else { + if err != nil { + t.Errorf("get data from store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err) + } + } + } + }) }) } From 181cb2396821feb5a46055737542237f9158e5b9 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Tue, 20 Nov 2018 10:15:13 +0100 Subject: [PATCH 06/16] swarm/shed: remove MockIndex --- swarm/shed/example_store_test.go | 30 ++------ swarm/shed/index.go | 89 ++++++------------------ swarm/shed/index_mock.go | 113 ------------------------------- swarm/shed/index_mock_test.go | 45 ------------ swarm/shed/index_test.go | 32 --------- 5 files changed, 23 insertions(+), 286 deletions(-) delete mode 100644 swarm/shed/index_mock.go delete mode 100644 swarm/shed/index_mock_test.go diff --git a/swarm/shed/example_store_test.go b/swarm/shed/example_store_test.go index d1e6aa463b5..563a98c8992 100644 --- a/swarm/shed/example_store_test.go +++ b/swarm/shed/example_store_test.go @@ -24,14 +24,10 @@ import ( "io/ioutil" "log" "os" - "strings" "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/swarm/shed" "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/storage/mock" - "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" "github.com/syndtr/goleveldb/leveldb" ) @@ -47,7 +43,7 @@ type Store struct { schemaName shed.StringField sizeCounter shed.Uint64Field accessCounter shed.Uint64Field - retrievalIndex shed.IndexInterface // example of swapable Index/MockIndex + retrievalIndex shed.Index accessIndex shed.Index gcIndex shed.Index } @@ -55,7 +51,7 @@ type Store struct { // New returns new Store. All fields and indexes are initialized // and possible conflicts with schema from existing database is checked // automatically. -func New(path string, mockStore *mock.NodeStore) (s *Store, err error) { +func New(path string) (s *Store, err error) { db, err := shed.NewDB(path) if err != nil { return nil, err @@ -74,7 +70,7 @@ func New(path string, mockStore *mock.NodeStore) (s *Store, err error) { return nil, err } // Index storing actual chunk address, data and store timestamp. - retrievalIndex, err := db.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{ + s.retrievalIndex, err = db.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{ EncodeKey: func(fields shed.IndexItem) (key []byte, err error) { return fields.Address, nil }, @@ -97,11 +93,6 @@ func New(path string, mockStore *mock.NodeStore) (s *Store, err error) { if err != nil { return nil, err } - s.retrievalIndex = retrievalIndex - if mockStore != nil { - // If mock store is provided, use it for retrieval index. - s.retrievalIndex = retrievalIndex.NewMockIndex(mockStore) - } // Index storing access timestamp for a particular address. // It is needed in order to update gc index keys for iteration order. s.accessIndex, err = db.NewIndex("Address->AccessTimestamp", shed.IndexFuncs{ @@ -318,20 +309,7 @@ func Example_store() { } defer os.RemoveAll(dir) - var mockStore *mock.NodeStore - // Configuring mock store using environment variable is - // just an example, it can be accomplished in more elegant ways. - if strings.EqualFold(os.Getenv("SWARM_USE_MOCKSTORE"), "true") { - // Global store is constructed here for example purposes. - // It should be available globally so that all nodes can access it. - globalStore := mem.NewGlobalStore() - // An arbitrary address is used for this example. - // In real situations this address should be the same as the Swarm - // node address. - mockStore = globalStore.NewNodeStore(common.HexToAddress("12345678")) - } - - s, err := New(dir, mockStore) + s, err := New(dir) if err != nil { log.Fatal(err) } diff --git a/swarm/shed/index.go b/swarm/shed/index.go index 04ed0f19c97..aa23dc9ecc1 100644 --- a/swarm/shed/index.go +++ b/swarm/shed/index.go @@ -17,7 +17,6 @@ package shed import ( - "github.com/ethereum/go-ethereum/swarm/storage/mock" "github.com/syndtr/goleveldb/leveldb" ) @@ -94,10 +93,25 @@ func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) { } prefix := []byte{id} return Index{ - db: db, - prefix: prefix, - encodeKeyFunc: newIndexEncodeKeyFunc(funcs.EncodeKey, id), - decodeKeyFunc: newDecodeKeyFunc(funcs.DecodeKey), + db: db, + prefix: prefix, + // This function adjusts Index LevelDB key + // by appending the provided index id byte. + // This is needed to avoid collisions between keys of different + // indexes as all index ids are unique. + encodeKeyFunc: func(e IndexItem) (key []byte, err error) { + key, err = funcs.EncodeKey(e) + if err != nil { + return nil, err + } + return append(append(make([]byte, 0, len(key)+1), prefix...), key...), nil + }, + // This function reverses the encodeKeyFunc constructed key + // to transparently work with index keys without their index ids. + // It assumes that index keys are prefixed with only one byte. + decodeKeyFunc: func(key []byte) (e IndexItem, err error) { + return funcs.DecodeKey(key[1:]) + }, encodeValueFunc: funcs.EncodeValue, decodeValueFunc: funcs.DecodeValue, }, nil @@ -242,68 +256,3 @@ func (f Index) IterateFrom(start IndexItem, fn IndexIterFunc) (err error) { } return it.Error() } - -// NewMockIndex is a helper function to easily construct MockIndex -// when from the same definition of Index. -func (f Index) NewMockIndex(store *mock.NodeStore) (m MockIndex) { - return MockIndex{ - store: store, - prefix: f.prefix, - encodeKeyFunc: f.encodeKeyFunc, - decodeKeyFunc: f.decodeKeyFunc, - encodeValueFunc: f.encodeValueFunc, - decodeValueFunc: f.decodeValueFunc, - } -} - -// IndexInterface defines methods that are required for a simple index -// that does not use iterations. -// It can be used when Index should be swapped with MockIndex or any other -// IndexInterface implementation. -// In most cases, interface is not needed and it is recommended to use the -// Index implementation whenever possible. -type IndexInterface interface { - Get(keyFields IndexItem) (out IndexItem, err error) - Put(i IndexItem) (err error) - PutInBatch(batch *leveldb.Batch, i IndexItem) (err error) - Delete(keyFields IndexItem) (err error) - DeleteInBatch(batch *leveldb.Batch, keyFields IndexItem) (err error) -} - -// IndexIteratorInterface defines metods for a full index implementation with -// iterators. Index type implements this type. -// In most cases, interface is not needed and it is recommended to use the -// Index implementation whenever possible. -type IndexIteratorInterface interface { - IndexInterface - IterateAll(fn IndexIterFunc) (err error) - IterateFrom(start IndexItem, fn IndexIterFunc) (err error) -} - -// newIndexEncodeKeyFunc adjusts Index and MockIndex LevelDB key -// by appending the provided index id byte. -// This is needed to avoid collisions between keys of different -// indexes as all index ids are unique. -func newIndexEncodeKeyFunc( - encodeKeyFunc func(fields IndexItem) (key []byte, err error), - id byte, -) (f func(e IndexItem) (key []byte, err error)) { - prefix := []byte{id} - return func(e IndexItem) (key []byte, err error) { - key, err = encodeKeyFunc(e) - if err != nil { - return nil, err - } - return append(append(make([]byte, 0, len(key)+1), prefix...), key...), nil - } -} - -// newDecodeKeyFunc reverses the newIndexEncodeKeyFunc constructd key -// to transparently work with index keys without their index ids. -// This function is used in NewIndex and NewMockIndex constructors. -// It assumes that index keys are prefixed with only one byte. -func newDecodeKeyFunc(decodeKeyFunc func(key []byte) (e IndexItem, err error)) (f func(key []byte) (e IndexItem, err error)) { - return func(key []byte) (e IndexItem, err error) { - return decodeKeyFunc(key[1:]) - } -} diff --git a/swarm/shed/index_mock.go b/swarm/shed/index_mock.go deleted file mode 100644 index d9b7187a3dc..00000000000 --- a/swarm/shed/index_mock.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package shed - -import ( - "github.com/ethereum/go-ethereum/swarm/storage/mock" - "github.com/syndtr/goleveldb/leveldb" -) - -// MockIndex provides a way to inject a mock.NodeStore to store -// data centrally instead on provided DB. DB is used just for schema -// validation and identifying byte prefix for the particular index -// that is mocked. -// Iterator functions are not implemented and MockIndex can not replace -// indexes that rely on them. -// It implements IndexField interface. -type MockIndex struct { - store *mock.NodeStore - prefix []byte - encodeKeyFunc func(fields IndexItem) (key []byte, err error) - decodeKeyFunc func(key []byte) (e IndexItem, err error) - encodeValueFunc func(fields IndexItem) (value []byte, err error) - decodeValueFunc func(value []byte) (e IndexItem, err error) -} - -// NewMockIndex returns a new MockIndex instance with defined name and -// encoding functions. The name must be unique and will be validated -// on database schema for a key prefix byte. -// The data will not be saved on the DB itself, but using a provided -// mock.NodeStore. -func (db *DB) NewMockIndex(store *mock.NodeStore, name string, funcs IndexFuncs) (f MockIndex, err error) { - id, err := db.schemaIndexPrefix(name) - if err != nil { - return f, err - } - return MockIndex{ - store: store, - prefix: []byte{id}, - encodeKeyFunc: newIndexEncodeKeyFunc(funcs.EncodeKey, id), - decodeKeyFunc: newDecodeKeyFunc(funcs.DecodeKey), - encodeValueFunc: funcs.EncodeValue, - decodeValueFunc: funcs.DecodeValue, - }, nil -} - -// Get accepts key fields represented as IndexItem to retrieve a -// value from the index and return maximum available information -// from the index represented as another IndexItem. -func (f MockIndex) Get(keyFields IndexItem) (out IndexItem, err error) { - key, err := f.encodeKeyFunc(keyFields) - if err != nil { - return out, err - } - value, err := f.store.Get(key) - if err != nil { - return out, err - } - out, err = f.decodeValueFunc(value) - if err != nil { - return out, err - } - return out.Join(keyFields), nil -} - -// Put accepts IndexItem to encode information from it -// and save it to the database. -func (f MockIndex) Put(i IndexItem) (err error) { - key, err := f.encodeKeyFunc(i) - if err != nil { - return err - } - value, err := f.encodeValueFunc(i) - if err != nil { - return err - } - return f.store.Put(key, value) -} - -// PutInBatch is the same as Put method. -// Batch is ignored and the data is saved to the mock store instantly. -func (f MockIndex) PutInBatch(_ *leveldb.Batch, i IndexItem) (err error) { - return f.Put(i) -} - -// Delete accepts IndexItem to remove a key/value pair -// form the database based on its fields. -func (f MockIndex) Delete(keyFields IndexItem) (err error) { - key, err := f.encodeKeyFunc(keyFields) - if err != nil { - return err - } - return f.store.Delete(key) -} - -// DeleteInBatch is the same as Delete just the operation. -// Batch is ignored and the data is deleted on the mock store instantly. -func (f MockIndex) DeleteInBatch(_ *leveldb.Batch, keyFields IndexItem) (err error) { - return f.Delete(keyFields) -} diff --git a/swarm/shed/index_mock_test.go b/swarm/shed/index_mock_test.go deleted file mode 100644 index adc9dd068fc..00000000000 --- a/swarm/shed/index_mock_test.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package shed - -import ( - "testing" - - "github.com/ethereum/go-ethereum/common" - - "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" -) - -// TestMockIndex validates put, get and delete functions of -// the MockIndex implementation. -func TestMockIndex(t *testing.T) { - db, cleanupFunc := newTestDB(t) - defer cleanupFunc() - - globalStore := mem.NewGlobalStore() - - index, err := db.NewMockIndex( - globalStore.NewNodeStore(common.HexToAddress("12345678")), - "retrieval", - retrievalIndexFuncs, - ) - if err != nil { - t.Fatal(err) - } - - testIndex(t, db, index) -} diff --git a/swarm/shed/index_test.go b/swarm/shed/index_test.go index a5c81091cc1..f36ab7a3836 100644 --- a/swarm/shed/index_test.go +++ b/swarm/shed/index_test.go @@ -24,9 +24,6 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/swarm/storage/mock" - "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" "github.com/syndtr/goleveldb/leveldb" ) @@ -62,29 +59,6 @@ func TestIndex(t *testing.T) { t.Fatal(err) } - testIndex(t, db, index) -} - -// TestIndex_NewMockIndex validates put, get and delete functions of -// the MockIndex implementation, when constructed with Index.NewMockIndex function. -func TestIndex_NewMockIndex(t *testing.T) { - db, cleanupFunc := newTestDB(t) - defer cleanupFunc() - - index, err := db.NewIndex("retrieval", retrievalIndexFuncs) - if err != nil { - t.Fatal(err) - } - - globalStore := mem.NewGlobalStore() - - mockIndex := index.NewMockIndex(globalStore.NewNodeStore(common.HexToAddress("12345678"))) - - testIndex(t, db, mockIndex) -} - -// testIndex validates put, get and delete functions of a index interface. -func testIndex(t *testing.T, db *DB, index IndexInterface) { t.Run("put", func(t *testing.T) { want := IndexItem{ Address: []byte("put-hash"), @@ -196,9 +170,6 @@ func testIndex(t *testing.T, db *DB, index IndexInterface) { } wantErr := leveldb.ErrNotFound - if _, ok := index.(MockIndex); ok { - wantErr = mock.ErrNotFound - } got, err = index.Get(IndexItem{ Address: want.Address, }) @@ -236,9 +207,6 @@ func testIndex(t *testing.T, db *DB, index IndexInterface) { } wantErr := leveldb.ErrNotFound - if _, ok := index.(MockIndex); ok { - wantErr = mock.ErrNotFound - } got, err = index.Get(IndexItem{ Address: want.Address, }) From abc01688f4f6dee2df736c3706df9dd441e657dd Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Tue, 20 Nov 2018 10:21:41 +0100 Subject: [PATCH 07/16] swarm/shed: add UseMockStore field to IndexItem --- swarm/shed/index.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/swarm/shed/index.go b/swarm/shed/index.go index aa23dc9ecc1..c4cf504bfe0 100644 --- a/swarm/shed/index.go +++ b/swarm/shed/index.go @@ -37,6 +37,9 @@ type IndexItem struct { Data []byte AccessTimestamp int64 StoreTimestamp int64 + // UseMockStore is a pointer to identify + // an unset state of the field in Join function. + UseMockStore *bool } // Join is a helper method to construct a new @@ -55,6 +58,9 @@ func (i IndexItem) Join(i2 IndexItem) (new IndexItem) { if i.StoreTimestamp == 0 { i.StoreTimestamp = i2.StoreTimestamp } + if i.UseMockStore == nil { + i.UseMockStore = i2.UseMockStore + } return i } From 56a2ab76c2c48929e126b6a2cfffd799285e5de5 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Wed, 21 Nov 2018 12:36:35 +0100 Subject: [PATCH 08/16] swarm/shed: correctly name DB.writebatch counter --- swarm/shed/db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/shed/db.go b/swarm/shed/db.go index 987c89dcff4..66935d5908d 100644 --- a/swarm/shed/db.go +++ b/swarm/shed/db.go @@ -99,7 +99,7 @@ func (db *DB) NewIterator() iterator.Iterator { // WriteBatch wraps LevelDB Write method to increment metrics counter. func (db *DB) WriteBatch(batch *leveldb.Batch) error { - metrics.GetOrRegisterCounter("DB.write", nil).Inc(1) + metrics.GetOrRegisterCounter("DB.writebatch", nil).Inc(1) return db.ldb.Write(batch, nil) } From 1e8f0ded9355af627dbf83966e292da96e3c3da0 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Fri, 23 Nov 2018 10:25:24 +0100 Subject: [PATCH 09/16] swarm/shed: fix typos --- swarm/shed/field_string.go | 2 +- swarm/shed/field_struct.go | 2 +- swarm/shed/index.go | 2 +- swarm/shed/schema.go | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/swarm/shed/field_string.go b/swarm/shed/field_string.go index c2e5ffa14a3..a7e8f0c7547 100644 --- a/swarm/shed/field_string.go +++ b/swarm/shed/field_string.go @@ -27,7 +27,7 @@ type StringField struct { key []byte } -// NewStringField retruns a new Instance fo StringField. +// NewStringField retruns a new Instance of StringField. // It validates its name and type against the database schema. func (db *DB) NewStringField(name string) (f StringField, err error) { key, err := db.schemaFieldKey(name, "string") diff --git a/swarm/shed/field_struct.go b/swarm/shed/field_struct.go index 93d6e940964..07b9454811f 100644 --- a/swarm/shed/field_struct.go +++ b/swarm/shed/field_struct.go @@ -41,7 +41,7 @@ func (db *DB) NewStructField(name string) (f StructField, err error) { }, nil } -// Unmarshal unmarshals data fromt he database to a provided val. +// Unmarshal unmarshals data fromt the database to a provided val. // If the data is not found leveldb.ErrNotFound is returned. func (f StructField) Unmarshal(val interface{}) (err error) { b, err := f.db.Get(f.key) diff --git a/swarm/shed/index.go b/swarm/shed/index.go index c4cf504bfe0..7b8c2ef7284 100644 --- a/swarm/shed/index.go +++ b/swarm/shed/index.go @@ -173,7 +173,7 @@ func (f Index) PutInBatch(batch *leveldb.Batch, i IndexItem) (err error) { } // Delete accepts IndexItem to remove a key/value pair -// form the database based on its fields. +// from the database based on its fields. func (f Index) Delete(keyFields IndexItem) (err error) { key, err := f.encodeKeyFunc(keyFields) if err != nil { diff --git a/swarm/shed/schema.go b/swarm/shed/schema.go index 579d058aba1..cfb7c6d6426 100644 --- a/swarm/shed/schema.go +++ b/swarm/shed/schema.go @@ -56,10 +56,10 @@ type indexSpec struct { // a particular field form the schema definition. func (db *DB) schemaFieldKey(name, fieldType string) (key []byte, err error) { if name == "" { - return nil, errors.New("filed name can not be blank") + return nil, errors.New("field name can not be blank") } if fieldType == "" { - return nil, errors.New("filed type can not be blank") + return nil, errors.New("field type can not be blank") } s, err := db.getSchema() if err != nil { From 7dfada1d97086a0b758cfc4b50917af7c3c61773 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Fri, 23 Nov 2018 10:26:02 +0100 Subject: [PATCH 10/16] swarm/shed: remove unused field from IndexItem --- swarm/shed/index.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/swarm/shed/index.go b/swarm/shed/index.go index 7b8c2ef7284..ac2bbe84311 100644 --- a/swarm/shed/index.go +++ b/swarm/shed/index.go @@ -37,9 +37,6 @@ type IndexItem struct { Data []byte AccessTimestamp int64 StoreTimestamp int64 - // UseMockStore is a pointer to identify - // an unset state of the field in Join function. - UseMockStore *bool } // Join is a helper method to construct a new @@ -58,9 +55,6 @@ func (i IndexItem) Join(i2 IndexItem) (new IndexItem) { if i.StoreTimestamp == 0 { i.StoreTimestamp = i2.StoreTimestamp } - if i.UseMockStore == nil { - i.UseMockStore = i2.UseMockStore - } return i } From aadd1d2f65f39f55c91439b0b4d9ea53a002797b Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Fri, 23 Nov 2018 10:30:54 +0100 Subject: [PATCH 11/16] swamr/shed: propagate error in example store close method. --- swarm/shed/example_store_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/swarm/shed/example_store_test.go b/swarm/shed/example_store_test.go index 563a98c8992..2ed0be1413b 100644 --- a/swarm/shed/example_store_test.go +++ b/swarm/shed/example_store_test.go @@ -297,8 +297,8 @@ func (s *Store) PutSchema(name string) (err error) { } // Close closes the underlying database. -func (s *Store) Close() { - s.db.Close() +func (s *Store) Close() error { + return s.db.Close() } // Example_store constructs a simple storage implementation using shed package. @@ -313,6 +313,7 @@ func Example_store() { if err != nil { log.Fatal(err) } + defer s.Close() ch := storage.GenerateRandomChunk(1024) err = s.Put(context.Background(), ch) From 895aadc0264409895fe427a91382d5f9ad77b957 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Fri, 23 Nov 2018 10:37:39 +0100 Subject: [PATCH 12/16] swarm/shed: rename IndexItem.Join to IndexItem.Merge --- swarm/shed/index.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/swarm/shed/index.go b/swarm/shed/index.go index ac2bbe84311..daac9bef742 100644 --- a/swarm/shed/index.go +++ b/swarm/shed/index.go @@ -39,10 +39,10 @@ type IndexItem struct { StoreTimestamp int64 } -// Join is a helper method to construct a new +// Merge is a helper method to construct a new // IndexItem by filling up fields with default values // of a particular IndexItem with values from another one. -func (i IndexItem) Join(i2 IndexItem) (new IndexItem) { +func (i IndexItem) Merge(i2 IndexItem) (new IndexItem) { if i.Address == nil { i.Address = i2.Address } @@ -133,7 +133,7 @@ func (f Index) Get(keyFields IndexItem) (out IndexItem, err error) { if err != nil { return out, err } - return out.Join(keyFields), nil + return out.Merge(keyFields), nil } // Put accepts IndexItem to encode information from it @@ -212,7 +212,7 @@ func (f Index) IterateAll(fn IndexIterFunc) (err error) { if err != nil { return err } - stop, err := fn(keyIndexItem.Join(valueIndexItem)) + stop, err := fn(keyIndexItem.Merge(valueIndexItem)) if err != nil { return err } @@ -246,7 +246,7 @@ func (f Index) IterateFrom(start IndexItem, fn IndexIterFunc) (err error) { if err != nil { return err } - stop, err := fn(keyIndexItem.Join(valueIndexItem)) + stop, err := fn(keyIndexItem.Merge(valueIndexItem)) if err != nil { return err } From 1f7dd164bc1d92a00c87e5d120b58cd9ee9a75c7 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Fri, 23 Nov 2018 10:38:30 +0100 Subject: [PATCH 13/16] swarm/shed: rename StructField.Unmarshal to StructField.Get --- swarm/shed/field_struct.go | 4 ++-- swarm/shed/field_struct_test.go | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/swarm/shed/field_struct.go b/swarm/shed/field_struct.go index 07b9454811f..90daee7fc4f 100644 --- a/swarm/shed/field_struct.go +++ b/swarm/shed/field_struct.go @@ -41,9 +41,9 @@ func (db *DB) NewStructField(name string) (f StructField, err error) { }, nil } -// Unmarshal unmarshals data fromt the database to a provided val. +// Get unmarshals data from the database to a provided val. // If the data is not found leveldb.ErrNotFound is returned. -func (f StructField) Unmarshal(val interface{}) (err error) { +func (f StructField) Get(val interface{}) (err error) { b, err := f.db.Get(f.key) if err != nil { return err diff --git a/swarm/shed/field_struct_test.go b/swarm/shed/field_struct_test.go index 137ac3a0eda..cc0be01863f 100644 --- a/swarm/shed/field_struct_test.go +++ b/swarm/shed/field_struct_test.go @@ -22,7 +22,7 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) -// TestStructField validates put and unmarshal operations +// TestStructField validates put and get operations // of the StructField. func TestStructField(t *testing.T) { db, cleanupFunc := newTestDB(t) @@ -37,9 +37,9 @@ func TestStructField(t *testing.T) { A string } - t.Run("unmarshal empty", func(t *testing.T) { + t.Run("get empty", func(t *testing.T) { var s complexStructure - err := complexField.Unmarshal(&s) + err := complexField.Get(&s) if err != leveldb.ErrNotFound { t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound) } @@ -58,7 +58,7 @@ func TestStructField(t *testing.T) { t.Fatal(err) } var got complexStructure - err = complexField.Unmarshal(&got) + err = complexField.Get(&got) if err != nil { t.Fatal(err) } @@ -75,7 +75,7 @@ func TestStructField(t *testing.T) { t.Fatal(err) } var got complexStructure - err = complexField.Unmarshal(&got) + err = complexField.Get(&got) if err != nil { t.Fatal(err) } @@ -96,7 +96,7 @@ func TestStructField(t *testing.T) { t.Fatal(err) } var got complexStructure - err := complexField.Unmarshal(&got) + err := complexField.Get(&got) if err != nil { t.Fatal(err) } @@ -115,7 +115,7 @@ func TestStructField(t *testing.T) { t.Fatal(err) } var got complexStructure - err := complexField.Unmarshal(&got) + err := complexField.Get(&got) if err != nil { t.Fatal(err) } From 215a92f11c159718be21491e8267d5a49994358b Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Fri, 23 Nov 2018 10:55:42 +0100 Subject: [PATCH 14/16] swarm/shed: add metrics for db failures --- swarm/shed/db.go | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/swarm/shed/db.go b/swarm/shed/db.go index 66935d5908d..e128b8cbc86 100644 --- a/swarm/shed/db.go +++ b/swarm/shed/db.go @@ -71,23 +71,39 @@ func NewDB(path string) (db *DB, err error) { // Put wraps LevelDB Put method to increment metrics counter. func (db *DB) Put(key []byte, value []byte) (err error) { + err = db.ldb.Put(key, value, nil) + if err != nil { + metrics.GetOrRegisterCounter("DB.putFail", nil).Inc(1) + return err + } metrics.GetOrRegisterCounter("DB.put", nil).Inc(1) - - return db.ldb.Put(key, value, nil) + return nil } // Get wraps LevelDB Get method to increment metrics counter. func (db *DB) Get(key []byte) (value []byte, err error) { + value, err = db.ldb.Get(key, nil) + if err != nil { + if err == leveldb.ErrNotFound { + metrics.GetOrRegisterCounter("DB.getNotFound", nil).Inc(1) + } else { + metrics.GetOrRegisterCounter("DB.getFail", nil).Inc(1) + } + return nil, err + } metrics.GetOrRegisterCounter("DB.get", nil).Inc(1) - - return db.ldb.Get(key, nil) + return value, nil } // Delete wraps LevelDB Delete method to increment metrics counter. -func (db *DB) Delete(key []byte) error { +func (db *DB) Delete(key []byte) (err error) { + err = db.ldb.Delete(key, nil) + if err != nil { + metrics.GetOrRegisterCounter("DB.deleteFail", nil).Inc(1) + return err + } metrics.GetOrRegisterCounter("DB.delete", nil).Inc(1) - - return db.ldb.Delete(key, nil) + return nil } // NewIterator wraps LevelDB NewIterator method to increment metrics counter. @@ -98,10 +114,14 @@ func (db *DB) NewIterator() iterator.Iterator { } // WriteBatch wraps LevelDB Write method to increment metrics counter. -func (db *DB) WriteBatch(batch *leveldb.Batch) error { +func (db *DB) WriteBatch(batch *leveldb.Batch) (err error) { + err = db.ldb.Write(batch, nil) + if err != nil { + metrics.GetOrRegisterCounter("DB.writebatchFail", nil).Inc(1) + return err + } metrics.GetOrRegisterCounter("DB.writebatch", nil).Inc(1) - - return db.ldb.Write(batch, nil) + return nil } // Close closes LevelDB database. From e2a7c0cc73ea3551c722f4acc2b7d8849bfeea1a Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Fri, 23 Nov 2018 11:53:15 +0100 Subject: [PATCH 15/16] swarm/shed: add UseMockStore field to IndexItem --- swarm/shed/index.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/swarm/shed/index.go b/swarm/shed/index.go index daac9bef742..ba803e3c23b 100644 --- a/swarm/shed/index.go +++ b/swarm/shed/index.go @@ -37,6 +37,9 @@ type IndexItem struct { Data []byte AccessTimestamp int64 StoreTimestamp int64 + // UseMockStore is a pointer to identify + // an unset state of the field in Join function. + UseMockStore *bool } // Merge is a helper method to construct a new @@ -55,6 +58,9 @@ func (i IndexItem) Merge(i2 IndexItem) (new IndexItem) { if i.StoreTimestamp == 0 { i.StoreTimestamp = i2.StoreTimestamp } + if i.UseMockStore == nil { + i.UseMockStore = i2.UseMockStore + } return i } From fb2de8049ec767ee7e068b18809fdc84002606a2 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Sun, 25 Nov 2018 17:16:22 +0100 Subject: [PATCH 16/16] swarm/shed: add TestIndex/put_in_batch_twice test --- swarm/shed/index_test.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/swarm/shed/index_test.go b/swarm/shed/index_test.go index f36ab7a3836..ba82216dfeb 100644 --- a/swarm/shed/index_test.go +++ b/swarm/shed/index_test.go @@ -143,6 +143,40 @@ func TestIndex(t *testing.T) { }) }) + t.Run("put in batch twice", func(t *testing.T) { + // ensure that the last item of items with the same db keys + // is actually saved + batch := new(leveldb.Batch) + address := []byte("put-in-batch-twice-hash") + + // put the first item + index.PutInBatch(batch, IndexItem{ + Address: address, + Data: []byte("DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + }) + + want := IndexItem{ + Address: address, + Data: []byte("New DATA"), + StoreTimestamp: time.Now().UTC().UnixNano(), + } + // then put the item that will produce the same key + // but different value in the database + index.PutInBatch(batch, want) + db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err := index.Get(IndexItem{ + Address: address, + }) + if err != nil { + t.Fatal(err) + } + checkIndexItem(t, got, want) + }) + t.Run("delete", func(t *testing.T) { want := IndexItem{ Address: []byte("delete-hash"),