diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e7620b..e357901 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,5 +27,5 @@ jobs: # specify any bash command here prefixed with `run: ` - run: go get -v -t -d ./... - - run: go test -v -race -short -coverprofile=coverage.txt ./... + - run: go test -v -race -short -coverprofile=coverage.txt -timeout 1800s ./... - run: bash <(curl -s https://codecov.io/bash) \ No newline at end of file diff --git a/README.md b/README.md index b17d061..d7c769a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # go-datastores +[![codecov](https://codecov.io/gh/RTradeLtd/go-datastores/branch/master/graph/badge.svg)](https://codecov.io/gh/RTradeLtd/go-datastores) [![Build Status](https://travis-ci.com/RTradeLtd/go-datastores.svg?branch=master)](https://travis-ci.com/RTradeLtd/go-datastores) [![RTradeLtd](https://circleci.com/gh/RTradeLtd/go-datastores.svg?style=shield)](https://app.circleci.com/pipelines/github/RTradeLtd/go-datastores) [![Go Report Card](https://goreportcard.com/badge/github.com/RTradeLtd/go-datastores)](https://goreportcard.com/report/github.com/RTradeLtd/go-datastores) + + `go-datastores` is a collection of a variety of IPFS datastores to be used by TemporalX in a single monorepo for easy maintenance. A majority of these datastores are forked from upstream repositories, with minor modifications to faciltiate easier integration with TemporalX, along with performance improvements and optimizations where possible. Additionally it allows us to pull in all datastores we need from a single repository. If you are a user of TemporalX and want to be able to use datastores that we do not yet support, you can submit a PR and we'll enable usage of the datastore within our next release diff --git a/badger/datastore.go b/badger/datastore.go index 254d872..9ef5861 100644 --- a/badger/datastore.go +++ b/badger/datastore.go @@ -1,7 +1,6 @@ package dsbadger import ( - "context" "errors" "log" "time" @@ -519,169 +518,6 @@ func (t *txn) Query(q dsq.Query) (dsq.Results, error) { return t.query(q) } -func (t *txn) query(q dsq.Query) (dsq.Results, error) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - opt := badger.DefaultIteratorOptions - opt.PrefetchValues = !q.KeysOnly - prefix := ds.NewKey(q.Prefix).String() - if prefix != "/" { - opt.Prefix = []byte(prefix + "/") - } - // Handle ordering - if len(q.Orders) > 0 { - switch q.Orders[0].(type) { - case dsq.OrderByKey, *dsq.OrderByKey: - // We order by key by default. - case dsq.OrderByKeyDescending, *dsq.OrderByKeyDescending: - // Reverse order by key - opt.Reverse = true - default: - // Ok, we have a weird order we can't handle. Let's - // perform the _base_ query (prefix, filter, etc.), then - // handle sort/offset/limit later. - - // Skip the stuff we can't apply. - baseQuery := q - baseQuery.Limit = 0 - baseQuery.Offset = 0 - baseQuery.Orders = nil - - // perform the base query. - res, err := t.query(baseQuery) - if err != nil { - return nil, err - } - - // fix the query - res = dsq.ResultsReplaceQuery(res, q) - - // Remove the parts we've already applied. - naiveQuery := q - naiveQuery.Prefix = "" - naiveQuery.Filters = nil - - // Apply the rest of the query - return dsq.NaiveQueryApply(naiveQuery, res), nil - } - } - var ( - it = t.txn.NewIterator(opt) - done = make(chan bool) - resultChan = make(chan dsq.Result) - entries = make([]dsq.Entry, 0) - ) - go func() { - defer func() { - done <- true - }() - if t.ds.closed.IsSet() { - return - } - // this iterator is part of an implicit transaction, so when - // we're done we must discard the transaction. It's safe to - // discard the txn it because it contains the iterator only. - if t.implicit { - defer t.discard() - } - defer it.Close() - // All iterators must be started by rewinding. - it.Rewind() - // skip to the offset - for skipped := 0; skipped < q.Offset && it.Valid(); it.Next() { - // On the happy path, we have no filters and we can go - // on our way. - if len(q.Filters) == 0 { - skipped++ - continue - } - // On the sad path, we need to apply filters before - // counting the item as "skipped" as the offset comes - // _after_ the filter. - item := it.Item() - matches := true - check := func(value []byte) error { - e := dsq.Entry{Key: string(item.Key()), Value: value, Size: int(item.ValueSize())} - // Only calculate expirations if we need them. - if q.ReturnExpirations { - e.Expiration = expires(item) - } - matches = filter(q.Filters, e) - return nil - } - // Maybe check with the value, only if we need it. - var err error - if q.KeysOnly { - err = check(nil) - } else { - err = item.Value(check) - } - if err != nil { - select { - case resultChan <- dsq.Result{Error: err}: - case <-t.ds.closing: - return - case <-ctx.Done(): - return - } - } - if !matches { - skipped++ - } - } - for sent := 0; (q.Limit <= 0 || sent < q.Limit) && it.Valid(); it.Next() { - item := it.Item() - e := dsq.Entry{Key: string(item.Key())} - // Maybe get the value - var result dsq.Result - if !q.KeysOnly { - b, err := item.ValueCopy(nil) - if err != nil { - result = dsq.Result{Error: err} - } else { - e.Value = b - e.Size = len(b) - result = dsq.Result{Entry: e} - } - } else { - e.Size = int(item.ValueSize()) - result = dsq.Result{Entry: e} - } - - if q.ReturnExpirations { - result.Expiration = expires(item) - } - - // Finally, filter it (unless we're dealing with an error). - if result.Error == nil && filter(q.Filters, e) { - continue - } - - select { - case resultChan <- result: - sent++ - case <-ctx.Done(): - return - case <-t.ds.closing: - return - } - } - }() - for { - select { - case <-done: - goto FINISHED - case result := <-resultChan: - if result.Error != nil { - log.Println("query result failure: ", result.Error) - } - entries = append(entries, result.Entry) - } - } -FINISHED: - return dsq.ResultsWithEntries(q, entries), nil -} - func (t *txn) Commit() error { if t.ds.closed.IsSet() { return ErrClosed diff --git a/badger/ds_test.go b/badger/ds_test.go index 48730f4..0b8fe2c 100644 --- a/badger/ds_test.go +++ b/badger/ds_test.go @@ -10,23 +10,12 @@ import ( "testing" "time" + "github.com/RTradeLtd/go-datastores/testutils" ds "github.com/ipfs/go-datastore" dsq "github.com/ipfs/go-datastore/query" dstest "github.com/ipfs/go-datastore/test" ) -var testcases = map[string]string{ - "/a": "a", - "/a/b": "ab", - "/a/b/c": "abc", - "/a/b/d": "a/b/d", - "/a/c": "ac", - "/a/d": "ad", - "/e": "e", - "/f": "f", - "/g": "", -} - // returns datastore, and a function to call on exit. // (this garbage collects). So: // @@ -66,26 +55,6 @@ func newDSSync(t *testing.T, sync bool) (*Datastore, func()) { } } -func addTestCases(t *testing.T, d *Datastore, testcases map[string]string) { - for k, v := range testcases { - dsk := ds.NewKey(k) - if err := d.Put(dsk, []byte(v)); err != nil { - t.Fatal(err) - } - } - - for k, v := range testcases { - dsk := ds.NewKey(k) - v2, err := d.Get(dsk) - if err != nil { - t.Fatal(err) - } - if string(v2) != v { - t.Errorf("%s values differ: %s != %s", k, v, v2) - } - } -} - func Test_Sync(t *testing.T) { type args struct { sync bool @@ -129,12 +98,12 @@ func TestQuery(t *testing.T) { d, done := newDS(t) defer done() - addTestCases(t, d, testcases) + testutils.AddTestCases(t, d, testutils.TestCases) rs, err := d.Query(dsq.Query{Prefix: "/a/"}) if err != nil { t.Fatal(err) } - expectMatches(t, []string{ + testutils.ExpectMatches(t, []string{ "/a/b", "/a/b/c", "/a/b/d", @@ -147,7 +116,7 @@ func TestQuery(t *testing.T) { if err != nil { t.Fatal(err) } - expectMatches(t, []string{ + testutils.ExpectMatches(t, []string{ "/a/b/d", "/a/c", }, rs) @@ -156,7 +125,7 @@ func TestQuery(t *testing.T) { func TestHas(t *testing.T) { d, done := newDS(t) defer done() - addTestCases(t, d, testcases) + testutils.AddTestCases(t, d, testutils.TestCases) has, err := d.Has(ds.NewKey("/a/b/c")) if err != nil { @@ -180,14 +149,14 @@ func TestHas(t *testing.T) { func TestGetSize(t *testing.T) { d, done := newDS(t) defer done() - addTestCases(t, d, testcases) + testutils.AddTestCases(t, d, testutils.TestCases) size, err := d.GetSize(ds.NewKey("/a/b/c")) if err != nil { t.Error(err) } - if size != len(testcases["/a/b/c"]) { + if size != len(testutils.TestCases["/a/b/c"]) { t.Error("") } @@ -200,7 +169,7 @@ func TestGetSize(t *testing.T) { func TestNotExistGet(t *testing.T) { d, done := newDS(t) defer done() - addTestCases(t, d, testcases) + testutils.AddTestCases(t, d, testutils.TestCases) has, err := d.Has(ds.NewKey("/a/b/c/d")) if err != nil { @@ -227,7 +196,7 @@ func TestNotExistGet(t *testing.T) { func TestDelete(t *testing.T) { d, done := newDS(t) defer done() - addTestCases(t, d, testcases) + testutils.AddTestCases(t, d, testutils.TestCases) has, err := d.Has(ds.NewKey("/a/b/c")) if err != nil { @@ -270,28 +239,6 @@ func TestGetEmpty(t *testing.T) { } } -func expectMatches(t *testing.T, expect []string, actualR dsq.Results) { - actual, err := actualR.Rest() - if err != nil { - t.Error(err) - } - - if len(actual) != len(expect) { - t.Error("not enough", expect, actual) - } - for _, k := range expect { - found := false - for _, e := range actual { - if e.Key == k { - found = true - } - } - if !found { - t.Error(k, "not found") - } - } -} - func TestBatching(t *testing.T) { d, done := newDS(t) defer done() @@ -301,7 +248,7 @@ func TestBatching(t *testing.T) { t.Fatal(err) } - for k, v := range testcases { + for k, v := range testutils.TestCases { err := b.Put(ds.NewKey(k), []byte(v)) if err != nil { t.Fatal(err) @@ -313,7 +260,7 @@ func TestBatching(t *testing.T) { t.Fatal(err) } - for k, v := range testcases { + for k, v := range testutils.TestCases { val, err := d.Get(ds.NewKey(k)) if err != nil { t.Fatal(err) @@ -351,7 +298,7 @@ func TestBatching(t *testing.T) { t.Fatal(err) } - expectMatches(t, []string{ + testutils.ExpectMatches(t, []string{ "/a", "/a/b/d", "/a/c", @@ -656,7 +603,7 @@ func TestDiskUsage(t *testing.T) { if err != nil { t.Fatal(err) } - addTestCases(t, d, testcases) + testutils.AddTestCases(t, d, testutils.TestCases) d.Close() d, err = NewDatastore(path, nil) diff --git a/badger/query.go b/badger/query.go new file mode 100644 index 0000000..35deb22 --- /dev/null +++ b/badger/query.go @@ -0,0 +1,106 @@ +package dsbadger + +import ( + badger "github.com/dgraph-io/badger/v2" + ds "github.com/ipfs/go-datastore" + dsq "github.com/ipfs/go-datastore/query" +) + +/* + This file contians an optimize query function as the one included from upstream allocates a lot of memory + when parsing very large datasets. A key-value store with approximately 1.7TB of data stored requires a runtime allocation + of 2GB to simply iterate over all of its keys +*/ + +func (t *txn) query(q dsq.Query) (dsq.Results, error) { + if t.ds.closed.IsSet() { + return nil, ErrClosed + } + opt := badger.DefaultIteratorOptions + opt.PrefetchValues = !q.KeysOnly + prefix := ds.NewKey(q.Prefix).String() + if prefix != "/" { + opt.Prefix = []byte(prefix + "/") + } + // Handle ordering + if len(q.Orders) > 0 { + switch q.Orders[0].(type) { + case dsq.OrderByKey, *dsq.OrderByKey: + case dsq.OrderByKeyDescending, *dsq.OrderByKeyDescending: + opt.Reverse = true + default: + baseQuery := q + baseQuery.Limit = 0 + baseQuery.Offset = 0 + baseQuery.Orders = nil + res, err := t.query(baseQuery) + if err != nil { + return nil, err + } + res = dsq.ResultsReplaceQuery(res, q) + naiveQuery := q + naiveQuery.Prefix = "" + naiveQuery.Filters = nil + return dsq.NaiveQueryApply(naiveQuery, res), nil + } + } + var ( + it = t.txn.NewIterator(opt) + item *badger.Item + entries []dsq.Entry + ) + if t.implicit { + defer t.discard() + } + defer it.Close() + it.Rewind() + for skipped := 0; skipped < q.Offset && it.Valid(); it.Next() { + if len(q.Filters) == 0 { + skipped++ + continue + } + item = it.Item() + matches := true + check := func(value []byte) error { + e := dsq.Entry{Key: string(item.Key()), Value: value, Size: int(item.ValueSize())} + // Only calculate expirations if we need them. + if q.ReturnExpirations { + e.Expiration = expires(item) + } + matches = filter(q.Filters, e) + return nil + } + if q.KeysOnly { + _ = check(nil) + } else { + _ = item.Value(check) + } + if !matches { + skipped++ + } + } + var value []byte + for sent := 0; (q.Limit <= 0 || sent < q.Limit) && it.Valid(); it.Next() { + item = it.Item() + entry := dsq.Entry{Key: string(item.Key())} + var copyErr error + if !q.KeysOnly { + value, copyErr = item.ValueCopy(nil) + if copyErr == nil { + entry.Value = value + } + entry.Size = len(entry.Value) + } else { + entry.Size = int(item.ValueSize()) + } + if q.ReturnExpirations { + entry.Expiration = expires(item) + } + if copyErr == nil && filter(q.Filters, entry) { + continue + } + entries = append(entries, entry) + sent++ + } + return dsq.ResultsWithEntries(q, entries), nil +} diff --git a/leveldb/ds_test.go b/leveldb/ds_test.go index 7886859..bc827b0 100644 --- a/leveldb/ds_test.go +++ b/leveldb/ds_test.go @@ -8,23 +8,13 @@ import ( "sort" "testing" + "github.com/RTradeLtd/go-datastores/testutils" ds "github.com/ipfs/go-datastore" dsq "github.com/ipfs/go-datastore/query" dstest "github.com/ipfs/go-datastore/test" "github.com/ucwong/goleveldb/leveldb" ) -var testcases = map[string]string{ - "/a": "a", - "/a/b": "ab", - "/a/b/c": "abc", - "/a/b/d": "a/b/d", - "/a/c": "ac", - "/a/d": "ad", - "/e": "e", - "/f": "f", -} - // returns datastore, and a function to call on exit. // (this garbage collects). So: // @@ -44,36 +34,15 @@ func newDS(t *testing.T) (*Datastore, func()) { } } -func addTestCases(t *testing.T, d *Datastore, testcases map[string]string) { - for k, v := range testcases { - dsk := ds.NewKey(k) - if err := d.Put(dsk, []byte(v)); err != nil { - t.Fatal(err) - } - } - - for k, v := range testcases { - dsk := ds.NewKey(k) - v2, err := d.Get(dsk) - if err != nil { - t.Fatal(err) - } - if string(v2) != v { - t.Errorf("%s values differ: %s != %s", k, v, v2) - } - } - -} - func testQuery(t *testing.T, d *Datastore) { - addTestCases(t, d, testcases) + testutils.AddTestCases(t, d, testutils.TestCases) rs, err := d.Query(dsq.Query{Prefix: "/a/"}) if err != nil { t.Fatal(err) } - expectMatches(t, []string{ + testutils.ExpectMatches(t, []string{ "/a/b", "/a/b/c", "/a/b/d", @@ -88,7 +57,7 @@ func testQuery(t *testing.T, d *Datastore) { t.Fatal(err) } - expectMatches(t, []string{ + testutils.ExpectMatches(t, []string{ "/a/b/d", "/a/c", }, rs) @@ -100,13 +69,13 @@ func testQuery(t *testing.T, d *Datastore) { t.Fatal(err) } - keys := make([]string, 0, len(testcases)) - for k := range testcases { + keys := make([]string, 0, len(testutils.TestCases)) + for k := range testutils.TestCases { keys = append(keys, k) } sort.Strings(keys) - expectOrderedMatches(t, keys, rs) + testutils.ExpectKeyOrderMatches(t, rs, keys) rs, err = d.Query(dsq.Query{Orders: []dsq.Order{dsq.OrderByKeyDescending{}}}) if err != nil { @@ -118,7 +87,7 @@ func testQuery(t *testing.T, d *Datastore) { keys[i], keys[j] = keys[j], keys[i] } - expectOrderedMatches(t, keys, rs) + testutils.ExpectKeyOrderMatches(t, rs, keys) } func TestEmptyOpts(t *testing.T) { @@ -142,7 +111,7 @@ func TestQuery(t *testing.T) { func TestQueryRespectsProcess(t *testing.T) { d, close := newDS(t) defer close() - addTestCases(t, d, testcases) + testutils.AddTestCases(t, d, testutils.TestCases) } func TestCloseRace(t *testing.T) { @@ -161,7 +130,7 @@ func TestCloseRace(t *testing.T) { close() closeCh <- nil }() - for k := range testcases { + for k := range testutils.TestCases { tx.Get(ds.NewKey(k)) } tx.Commit() @@ -170,7 +139,7 @@ func TestCloseRace(t *testing.T) { func TestCloseSafety(t *testing.T) { d, close := newDS(t) - addTestCases(t, d, testcases) + testutils.AddTestCases(t, d, testutils.TestCases) tx, _ := d.NewTransaction(false) err := tx.Put(ds.NewKey("test"), []byte("test")) @@ -187,53 +156,13 @@ func TestCloseSafety(t *testing.T) { } } -func expectMatches(t *testing.T, expect []string, actualR dsq.Results) { - t.Helper() - actual, err := actualR.Rest() - if err != nil { - t.Error(err) - } - - if len(actual) != len(expect) { - t.Error("not enough", expect, actual) - } - for _, k := range expect { - found := false - for _, e := range actual { - if e.Key == k { - found = true - } - } - if !found { - t.Error(k, "not found") - } - } -} - -func expectOrderedMatches(t *testing.T, expect []string, actualR dsq.Results) { - t.Helper() - actual, err := actualR.Rest() - if err != nil { - t.Error(err) - } - - if len(actual) != len(expect) { - t.Error("not enough", expect, actual) - } - for i := range expect { - if expect[i] != actual[i].Key { - t.Errorf("expected %q, got %q", expect[i], actual[i].Key) - } - } -} - func testBatching(t *testing.T, d *Datastore) { b, err := d.Batch() if err != nil { t.Fatal(err) } - for k, v := range testcases { + for k, v := range testutils.TestCases { err := b.Put(ds.NewKey(k), []byte(v)) if err != nil { t.Fatal(err) @@ -245,7 +174,7 @@ func testBatching(t *testing.T, d *Datastore) { t.Fatal(err) } - for k, v := range testcases { + for k, v := range testutils.TestCases { val, err := d.Get(ds.NewKey(k)) if err != nil { t.Fatal(err) @@ -265,7 +194,7 @@ func TestBatching(t *testing.T) { func TestDiskUsage(t *testing.T) { d, done := newDS(t) - addTestCases(t, d, testcases) + testutils.AddTestCases(t, d, testutils.TestCases) du, err := d.DiskUsage() if err != nil { t.Fatal(err) diff --git a/sql/README.md b/sql/README.md new file mode 100644 index 0000000..4eaafac --- /dev/null +++ b/sql/README.md @@ -0,0 +1,88 @@ +# SQL Datastore + +`sqlds` allows using arbitrary SQL databases as an IPFS datastore. It defines a set of functionality that can be used by any golang client that satisfies the `database/sql` interface. Included in an implementation for the PostgreSQL database. This datastore is forked from `ipfs/go-ds-sql` with a few modifications, and cleanup to the codebase, consisting of: + +* Enable table recreation, migration, and index creation as an optional capability when using PostgreSQL +* Change testing scheme to ensure that the postgres functionality is tested and subsequently the datastore implementation is tested + * Upstream doesn't specifically test the postgres implementation which is likely to introduce bugs. +* Use the `uber-go/multierr` package to combine errors where appropriatey + * In transaction mode if any errors happen, the error returned from the rollback was not returned, instead we combine the error for better visibility + +# Usage (Postgres) + +## Easy (Automatic) + + +The `postgres` datastore has some helper utility to enable automatic table and index creation, as well as the ability to recreate the table by dropping it, and creating it. Note that this setting requires `RunMigrations` also set to true. If not the `CreateIndex` setting will be ignored. Additionally note that we will never delete tables unless specified. + +Note that we will never delete tables automatically, and tables will only be deleted when `RecreateTables` is set to true, and `AcceptRecreateWarning` is set to the value of `RecreateTables`. Additionally you must make sure to set `RunMigrations` otherwise we wont actually create the tables, and instead just delete the existing tables. + +The following options can be used to enable table creation, and index creation: + + +```Go +import ( + pgds "github.com/RTradeLtd/go-datastores/sql/postgres" + "github.com/ipfs/go-datastore" +) +opts := &pgds.Options{ + Host: "127.0.0.1", + Port: "5432", + User: "postgres", + Database: "datastores", + Password: "password123", + SSLMode: "disable", + Table: "blocks", + // this will drop the existing table + AcceptRecreateWarning: pgds.RecreateTables, + RunMigrations: true, + RecreateTables: true, + CreateIndex: true, +} +ds, err := opts.Create() +if err != nil { + log.Fatal(err) +} +if err := ds.Put( + datastore.NewKey("much test very wow"), + []byte("whose even going to read this anyways"), +); err != nil { + log.Fatal(err) +} +``` + +## Hard (Manual) + +Ensure a database is created and a table exists with `key` and `data` columns. For example, in PostgreSQL you can create a table with the following structure (replacing `table_name` with the name of the table the datastore will use - by default this is `blocks`): + +```sql +CREATE TABLE IF NOT EXISTS table_name (key TEXT NOT NULL UNIQUE, data BYTEA) +``` + +It's recommended to create an index on the `key` column that is optimised for prefix scans. For example, in PostgreSQL you can create a `text_pattern_ops` index on the table: + +```sql +CREATE INDEX IF NOT EXISTS table_name_key_text_pattern_ops_idx ON table_name (key text_pattern_ops) +``` + +Import and use in your application: + +```go +import ( + "database/sql" + "github.com/RTradeLtd/go-datastores/sql" + pg "github.com/RTradeLtd/go-datastores/sql/postgres" +) + +mydb, _ := sql.Open("yourdb", "yourdbparameters") + +// Implement the Queries interface for your SQL impl. +// ...or use the provided PostgreSQL queries +queries := pg.NewQueries("blocks") + +ds := sqlds.NewDatastore(mydb, queries) +``` + +# license + +As this is forked from upstream, it retains its existing license which can be found in `LICENSE.orig`. \ No newline at end of file diff --git a/sql/postgres/postgres.go b/sql/postgres/postgres.go index cdf423a..18166ac 100644 --- a/sql/postgres/postgres.go +++ b/sql/postgres/postgres.go @@ -10,17 +10,27 @@ import ( _ "github.com/lib/pq" //postgres driver ) +var ( + // RecreateTables indicates that you accept all consequences from dropping your table and recreating it + RecreateTables = "recreate tables" +) + // Options are the postgres datastore options, reexported here for convenience. type Options struct { - Host string - Port string - User string - Password string - Database string - Table string - SSLMode string - RunMigrations bool - RecreateTables bool + // AcceptRecreateWarning is used as a safety check to pevent accidental deletion of existing tables and data + // To accept the warning, you must set the value of this field to `recreate tables`, which can be done manually + // or via the usage of the public `RecreateTables` variable + AcceptRecreateWarning string + Host string + Port string + User string + Password string + Database string + Table string + SSLMode string + RunMigrations bool + RecreateTables bool + CreateIndex bool } // Queries are the postgres queries for a given table. @@ -106,7 +116,8 @@ func (opts *Options) Create() (*sqlds.Datastore, error) { if err != nil { return nil, err } - if opts.RecreateTables { + // only recreate the tables *IF* warning is accepted + if opts.RecreateTables && opts.AcceptRecreateWarning == RecreateTables { if _, err := db.Exec( "DROP TABLE IF EXISTS blocks", ); err != nil { @@ -119,6 +130,17 @@ func (opts *Options) Create() (*sqlds.Datastore, error) { ); err != nil { return nil, multierr.Combine(err, db.Close()) } + if opts.CreateIndex { + if _, err := db.Exec( + fmt.Sprintf( + "CREATE INDEX IF NOT EXISTS %s_key_text_pattern_ops_idx ON %s (key text_pattern_ops)", + opts.Table, opts.Table, + ), + ); err != nil { + return nil, multierr.Combine(err, db.Close()) + } + + } } return sqlds.NewDatastore(db, NewQueries(opts.Table)), nil } diff --git a/sql/postgres/postgres_test.go b/sql/postgres/postgres_test.go index c59e53f..dec4f71 100644 --- a/sql/postgres/postgres_test.go +++ b/sql/postgres/postgres_test.go @@ -11,32 +11,23 @@ import ( dstest "github.com/ipfs/go-datastore/test" sqlds "github.com/RTradeLtd/go-datastores/sql" + "github.com/RTradeLtd/go-datastores/testutils" "github.com/ipfs/go-datastore" dsq "github.com/ipfs/go-datastore/query" ) -var testcases = map[string]string{ - "/a": "a", - "/a/b": "ab", - "/a/b/c": "abc", - "/a/b/d": "a/b/d", - "/a/c": "ac", - "/a/d": "ad", - "/e": "e", - "/f": "f", - "/g": "", -} - func newDS(t *testing.T) (*sqlds.Datastore, func(t *testing.T)) { opts := &Options{ - Host: "127.0.0.1", - Port: "5432", - User: "postgres", - Database: "datastores", - Password: "password123", - SSLMode: "disable", - RunMigrations: true, - RecreateTables: true, + Host: "127.0.0.1", + Port: "5432", + User: "postgres", + Database: "datastores", + Password: "password123", + SSLMode: "disable", + AcceptRecreateWarning: RecreateTables, + RunMigrations: true, + RecreateTables: true, + CreateIndex: true, } ds, err := opts.Create() if err != nil { @@ -49,27 +40,6 @@ func newDS(t *testing.T) (*sqlds.Datastore, func(t *testing.T)) { } } -func addTestCases(t *testing.T, d *sqlds.Datastore, testcases map[string]string) { - for k, v := range testcases { - dsk := datastore.NewKey(k) - if err := d.Put(dsk, []byte(v)); err != nil { - t.Fatal(err) - } - } - - for k, v := range testcases { - dsk := datastore.NewKey(k) - v2, err := d.Get(dsk) - if err != nil { - t.Fatal(err) - } - v2b := v2 - if string(v2b) != v { - t.Errorf("%s values differ: %s != %s", k, v, v2) - } - } -} - func TestPostgres_Queries(t *testing.T) { tableName := "querytabletest" queries := NewQueries(tableName) @@ -159,14 +129,14 @@ func TestQuery(t *testing.T) { d, done := newDS(t) defer done(t) - addTestCases(t, d, testcases) + testutils.AddTestCases(t, d, testutils.TestCases) // test prefix rs, err := d.Query(dsq.Query{Prefix: "/a/"}) if err != nil { t.Fatal(err) } - expectMatches(t, []string{ + testutils.ExpectMatches(t, []string{ "/a/b", "/a/b/c", "/a/b/d", @@ -179,7 +149,7 @@ func TestQuery(t *testing.T) { if err != nil { t.Fatal(err) } - expectMatches(t, []string{ + testutils.ExpectMatches(t, []string{ "/a/b/d", "/a/c", }, rs) @@ -191,7 +161,7 @@ func TestQuery(t *testing.T) { if err != nil { t.Fatal(err) } - expectKeyOrderMatches(t, rs, []string{ + testutils.ExpectKeyOrderMatches(t, rs, []string{ "/a/b", "/a/b/c", "/a/b/d", @@ -205,7 +175,7 @@ func TestQuery(t *testing.T) { if err != nil { t.Fatal(err) } - expectKeyOrderMatches(t, rs, []string{ + testutils.ExpectKeyOrderMatches(t, rs, []string{ "/a/d", "/a/c", "/a/b/d", @@ -220,7 +190,7 @@ func TestQuery(t *testing.T) { if err != nil { t.Fatal(err) } - expectKeyFilterMatches(t, rs, []string{"/a/b"}) + testutils.ExpectKeyFilterMatches(t, rs, []string{"/a/b"}) greaterThanFilter := dsq.FilterKeyCompare{Op: dsq.GreaterThan, Key: "/a/b"} greaterThanFilters := []dsq.Filter{greaterThanFilter} @@ -228,7 +198,7 @@ func TestQuery(t *testing.T) { if err != nil { t.Fatal(err) } - expectKeyFilterMatches(t, rs, []string{ + testutils.ExpectKeyFilterMatches(t, rs, []string{ "/a/b/c", "/a/b/d", "/a/c", @@ -241,7 +211,7 @@ func TestQuery(t *testing.T) { if err != nil { t.Fatal(err) } - expectKeyFilterMatches(t, rs, []string{ + testutils.ExpectKeyFilterMatches(t, rs, []string{ "/a/b", "/a/b/c", }) @@ -250,7 +220,7 @@ func TestQuery(t *testing.T) { func TestHas(t *testing.T) { d, done := newDS(t) defer done(t) - addTestCases(t, d, testcases) + testutils.AddTestCases(t, d, testutils.TestCases) has, err := d.Has(datastore.NewKey("/a/b/c")) if err != nil { @@ -274,7 +244,7 @@ func TestHas(t *testing.T) { func TestNotExistGet(t *testing.T) { d, done := newDS(t) defer done(t) - addTestCases(t, d, testcases) + testutils.AddTestCases(t, d, testutils.TestCases) has, err := d.Has(datastore.NewKey("/a/b/c/d")) if err != nil { @@ -301,7 +271,7 @@ func TestNotExistGet(t *testing.T) { func TestDelete(t *testing.T) { d, done := newDS(t) defer done(t) - addTestCases(t, d, testcases) + testutils.AddTestCases(t, d, testutils.TestCases) has, err := d.Has(datastore.NewKey("/a/b/c")) if err != nil { @@ -353,7 +323,7 @@ func TestBatching(t *testing.T) { t.Fatal(err) } - for k, v := range testcases { + for k, v := range testutils.TestCases { err := b.Put(datastore.NewKey(k), []byte(v)) if err != nil { t.Fatal(err) @@ -365,7 +335,7 @@ func TestBatching(t *testing.T) { t.Fatal(err) } - for k, v := range testcases { + for k, v := range testutils.TestCases { val, err := d.Get(datastore.NewKey(k)) if err != nil { t.Fatal(err) @@ -402,7 +372,7 @@ func TestBatching(t *testing.T) { t.Fatal(err) } - expectMatches(t, []string{ + testutils.ExpectMatches(t, []string{ "/a", "/a/b/d", "/a/c", @@ -810,70 +780,3 @@ func TestSuite(t *testing.T) { defer done(t) dstest.SubtestAll(t, d) } - -func expectKeyFilterMatches(t *testing.T, actual dsq.Results, expect []string) { - t.Helper() - actualE, err := actual.Rest() - if err != nil { - t.Error(err) - return - } - actualS := make([]string, len(actualE)) - for i, e := range actualE { - actualS[i] = e.Key - } - - if len(actualS) != len(expect) { - t.Error("length doesn't match.", expect, actualS) - return - } - - if strings.Join(actualS, "") != strings.Join(expect, "") { - t.Error("expect != actual.", expect, actualS) - return - } -} - -func expectMatches(t *testing.T, expect []string, actualR dsq.Results) { - t.Helper() - actual, err := actualR.Rest() - if err != nil { - t.Error(err) - } - - if len(actual) != len(expect) { - t.Error("not enough", expect, actual) - } - for _, k := range expect { - found := false - for _, e := range actual { - if e.Key == k { - found = true - } - } - if !found { - t.Error(k, "not found") - } - } -} - -func expectKeyOrderMatches(t *testing.T, actual dsq.Results, expect []string) { - t.Helper() - rs, err := actual.Rest() - if err != nil { - t.Error("error fetching dsq.Results", expect, actual) - return - } - - if len(rs) != len(expect) { - t.Error("expect != actual.", expect, actual) - return - } - - for i, r := range rs { - if r.Key != expect[i] { - t.Error("expect != actual.", expect, actual) - return - } - } -} diff --git a/testutils/testutil.go b/testutils/testutil.go new file mode 100644 index 0000000..ba6dfc8 --- /dev/null +++ b/testutils/testutil.go @@ -0,0 +1,116 @@ +package testutils + +import ( + "strings" + "testing" + + "github.com/ipfs/go-datastore" + dsq "github.com/ipfs/go-datastore/query" +) + +var ( + // TestCases are key-values to use in datastore testing + TestCases = map[string]string{ + "/a": "a", + "/a/b": "ab", + "/a/b/c": "abc", + "/a/b/d": "a/b/d", + "/a/c": "ac", + "/a/d": "ad", + "/e": "e", + "/f": "f", + "/g": "", + } +) + +// AddTestCases stores the key-value pairs in a datastore for testing +func AddTestCases(t *testing.T, d datastore.Datastore, testcases map[string]string) { + t.Helper() + for k, v := range testcases { + dsk := datastore.NewKey(k) + if err := d.Put(dsk, []byte(v)); err != nil { + t.Fatal(err) + } + } + + for k, v := range testcases { + dsk := datastore.NewKey(k) + v2, err := d.Get(dsk) + if err != nil { + t.Fatal(err) + } + if string(v2) != v { + t.Errorf("%s values differ: %s != %s", k, v, v2) + } + } +} + +// ExpectMatches is used to match expected results with actual results +func ExpectMatches(t *testing.T, expect []string, actualR dsq.Results) { + t.Helper() + actual, err := actualR.Rest() + if err != nil { + t.Error(err) + } + + if len(actual) != len(expect) { + t.Error("not enough", expect, actual) + } + for _, k := range expect { + found := false + for _, e := range actual { + if e.Key == k { + found = true + } + } + if !found { + t.Error(k, "not found") + } + } +} + +// ExpectKeyFilterMatches is used to verify filtering results +func ExpectKeyFilterMatches(t *testing.T, actual dsq.Results, expect []string) { + t.Helper() + actualE, err := actual.Rest() + if err != nil { + t.Error(err) + return + } + actualS := make([]string, len(actualE)) + for i, e := range actualE { + actualS[i] = e.Key + } + + if len(actualS) != len(expect) { + t.Error("length doesn't match.", expect, actualS) + return + } + + if strings.Join(actualS, "") != strings.Join(expect, "") { + t.Error("expect != actual.", expect, actualS) + return + } +} + +// ExpectKeyOrderMatches is used to verify ordering results +func ExpectKeyOrderMatches(t *testing.T, actual dsq.Results, expect []string) { + t.Helper() + rs, err := actual.Rest() + if err != nil { + t.Error("error fetching dsq.Results", expect, actual) + return + } + + if len(rs) != len(expect) { + t.Error("expect != actual.", expect, actual) + return + } + + for i, r := range rs { + if r.Key != expect[i] { + t.Error("expect != actual.", expect, actual) + return + } + } +}