Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Datastore copy util #469

Merged
merged 6 commits into from
Dec 9, 2020
+280 −28
Merged

Datastore copy util #469

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
bdd8cab
copy: adds a datastore copy util
sanderpick Dec 8, 2020
dca7247
dscopy: PR feedback, move out of datastore dir
sanderpick Dec 8, 2020
2e427a5
dscopy: log count every batch
sanderpick Dec 8, 2020
821769a
dscopy: make tool specific to go-threads
sanderpick Dec 9, 2020
b3f3ce3
dscopy: BadgerRepo -> BadgerRepos
sanderpick Dec 9, 2020
9d9b807
dscopy: move key lock
sanderpick Dec 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions common/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func DefaultNetwork(opts ...NetOption) (NetBoostrapper, error) {
}
fin.Add(pstore)

hostKey, err := getHostKey(config, litestore)
hostKey, err := getIPFSHostKey(config, litestore)
if err != nil {
return nil, fin.Cleanup(err)
}
Expand Down Expand Up @@ -174,12 +174,12 @@ func mongoStore(ctx context.Context, uri, db, collection string, fin *util.Final
return dstore, nil
}

func getHostKey(config NetConfig, store ds.Batching) (crypto.PrivKey, error) {
func getIPFSHostKey(config NetConfig, store ds.Datastore) (crypto.PrivKey, error) {
if len(config.MongoUri) != 0 {
k := ds.NewKey("key")
bytes, err := store.Get(k)
if errors.Is(err, ds.ErrNotFound) {
key, bytes, err := newHostKey()
key, bytes, err := newIPFSHostKey()
if err != nil {
return nil, err
}
Expand All @@ -194,16 +194,16 @@ func getHostKey(config NetConfig, store ds.Batching) (crypto.PrivKey, error) {
} else {
// If a local datastore is used, the key is written to a file
dir := filepath.Join(config.BadgerRepoPath, "ipfslite")
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
return nil, err
}
pth := filepath.Join(dir, "key")
_, err := os.Stat(pth)
if os.IsNotExist(err) {
key, bytes, err := newHostKey()
key, bytes, err := newIPFSHostKey()
if err != nil {
return nil, err
}
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
return nil, err
}
if err = ioutil.WriteFile(pth, bytes, 0400); err != nil {
return nil, err
}
Expand All @@ -220,7 +220,7 @@ func getHostKey(config NetConfig, store ds.Batching) (crypto.PrivKey, error) {
}
}

func newHostKey() (crypto.PrivKey, []byte, error) {
func newIPFSHostKey() (crypto.PrivKey, []byte, error) {
priv, _, err := crypto.GenerateKeyPair(crypto.Ed25519, 0)
if err != nil {
return nil, nil, err
Expand Down
2 changes: 1 addition & 1 deletion db/bench_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func createBenchDB(b *testing.B, opts ...NewOption) (*DB, func()) {
common.WithNetDebug(true),
)
checkBenchErr(b, err)
store, err := util.NewBadgerDatastore(dir, false)
store, err := util.NewBadgerDatastore(dir, "eventstore", false)
checkBenchErr(b, err)
d, err := NewDB(context.Background(), store, n, thread.NewIDV1(thread.Raw, 32), opts...)
checkBenchErr(b, err)
Expand Down
10 changes: 5 additions & 5 deletions db/db_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func TestE2EWithThreads(t *testing.T) {
checkErr(t, err)
defer n1.Close()

store, err := util.NewBadgerDatastore(tmpDir1, false)
store, err := util.NewBadgerDatastore(tmpDir1, "eventstore", false)
checkErr(t, err)
defer store.Close()

Expand Down Expand Up @@ -92,7 +92,7 @@ func TestE2EWithThreads(t *testing.T) {
Schema: util.SchemaFromInstance(&dummy{}, false),
}

store2, err := util.NewBadgerDatastore(tmpDir2, false)
store2, err := util.NewBadgerDatastore(tmpDir2, "eventstore", false)
checkErr(t, err)
defer store2.Close()

Expand Down Expand Up @@ -139,7 +139,7 @@ func TestMissingCollection(t *testing.T) {
checkErr(t, err)
defer n.Close()

store, err := util.NewBadgerDatastore(tmpDir, false)
store, err := util.NewBadgerDatastore(tmpDir, "eventstore", false)
checkErr(t, err)
defer store.Close()

Expand Down Expand Up @@ -175,7 +175,7 @@ func TestWithNewName(t *testing.T) {
)
checkErr(t, err)

store, err := util.NewBadgerDatastore(tmpDir, false)
store, err := util.NewBadgerDatastore(tmpDir, "eventstore", false)
checkErr(t, err)
defer store.Close()

Expand Down Expand Up @@ -223,7 +223,7 @@ func TestWithNewEventCodec(t *testing.T) {
)
checkErr(t, err)

store, err := util.NewBadgerDatastore(tmpDir, false)
store, err := util.NewBadgerDatastore(tmpDir, "eventstore", false)
checkErr(t, err)
defer store.Close()

Expand Down
3 changes: 1 addition & 2 deletions db/index.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,10 @@ import (
"fmt"
"sort"

dse "github.com/textileio/go-datastore-extensions"

"github.com/alecthomas/jsonschema"
ds "github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/query"
dse "github.com/textileio/go-datastore-extensions"
"github.com/tidwall/gjson"
"github.com/tidwall/sjson"
)
Expand Down
4 changes: 2 additions & 2 deletions db/manager_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ func TestManager_GetDB(t *testing.T) {
common.WithNetDebug(true),
)
checkErr(t, err)
store, err := util.NewBadgerDatastore(dir, false)
store, err := util.NewBadgerDatastore(dir, "eventstore", false)
checkErr(t, err)
man, err := NewManager(store, n, WithNewDebug(true))
checkErr(t, err)
Expand Down Expand Up @@ -256,7 +256,7 @@ func createTestManager(t *testing.T) (*Manager, func()) {
common.WithNetDebug(true),
)
checkErr(t, err)
store, err := util.NewBadgerDatastore(dir, false)
store, err := util.NewBadgerDatastore(dir, "eventstore", false)
checkErr(t, err)
m, err := NewManager(store, n, WithNewDebug(true))
checkErr(t, err)
Expand Down
2 changes: 1 addition & 1 deletion db/testutils_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ func createTestDB(t *testing.T, opts ...NewOption) (*DB, func()) {
common.WithNetDebug(true),
)
checkErr(t, err)
store, err := util.NewBadgerDatastore(dir, false)
store, err := util.NewBadgerDatastore(dir, "eventstore", false)
checkErr(t, err)
d, err := NewDB(context.Background(), store, n, thread.NewIDV1(thread.Raw, 32), opts...)
checkErr(t, err)
Expand Down
4 changes: 2 additions & 2 deletions integrationtests/foldersync/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func newRootClient(name, folderPath, repoPath string) (*client, error) {
return nil, err
}

s, err := util.NewBadgerDatastore(repoPath, false)
s, err := util.NewBadgerDatastore(repoPath, "eventstore", false)
if err != nil {
return nil, err
}
Expand All @@ -106,7 +106,7 @@ func newJoinerClient(name, folderPath, repoPath string, addr ma.Multiaddr, key t
return nil, err
}

s, err := util.NewBadgerDatastore(repoPath, false)
s, err := util.NewBadgerDatastore(repoPath, "eventstore", false)
if err != nil {
return nil, err
}
Expand Down
8 changes: 4 additions & 4 deletions integrationtests/foldersync/foldersync_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func TestSimple(t *testing.T) {
network0, err := newNetwork(repoPath0)
checkErr(t, err)

store0, err := util.NewBadgerDatastore(repoPath0, false)
store0, err := util.NewBadgerDatastore(repoPath0, "eventstore", false)
checkErr(t, err)
defer store0.Close()
db0, err := db.NewDB(context.Background(), store0, network0, id, db.WithNewCollections(cc))
Expand All @@ -63,7 +63,7 @@ func TestSimple(t *testing.T) {
network1, err := newNetwork(repoPath1)
checkErr(t, err)

store1, err := util.NewBadgerDatastore(repoPath1, false)
store1, err := util.NewBadgerDatastore(repoPath1, "eventstore", false)
checkErr(t, err)
defer store1.Close()
db1, err := db.NewDBFromAddr(
Expand All @@ -87,7 +87,7 @@ func TestSimple(t *testing.T) {
network2, err := newNetwork(repoPath2)
checkErr(t, err)

store2, err := util.NewBadgerDatastore(repoPath2, false)
store2, err := util.NewBadgerDatastore(repoPath2, "eventstore", false)
checkErr(t, err)
defer store2.Close()
db2, err := db.NewDBFromAddr(
Expand All @@ -111,7 +111,7 @@ func TestSimple(t *testing.T) {
network3, err := newNetwork(repoPath3)
checkErr(t, err)

store3, err := util.NewBadgerDatastore(repoPath3, false)
store3, err := util.NewBadgerDatastore(repoPath3, "eventstore", false)
checkErr(t, err)
defer store3.Close()
db3, err := db.NewDBFromAddr(
Expand Down
2 changes: 1 addition & 1 deletion threadsd/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ func main() {
if *mongoUri != "" {
store, err = mongods.New(ctx, *mongoUri, *mongoDatabase, mongods.WithCollName("eventstore"))
} else {
store, err = util.NewBadgerDatastore(*repo, *badgerLowMem)
store, err = util.NewBadgerDatastore(*repo, "eventstore", *badgerLowMem)
}
if err != nil {
log.Fatal(err)
Expand Down
Loading