Merge pull request #157 from ipfs/feat/fuzzflatfs

support flatfs fuzzing

fuzz/cmd/generate/main.go

@@ -64,7 +64,7 @@ import prov "{{ .Package }}"
 import ds "github.com/ipfs/go-datastore"
 
 func init() {
-	AddOpener("{{ .PackageName }}", func(loc string) ds.TxnDatastore {
+	AddOpener("{{ .PackageName }}", func(loc string) ds.Datastore {
 		d, err := prov.NewDatastore(loc, nil)
 		if err != nil {
 			panic("could not create db instance")

fuzz/fuzzer.go

@@ -16,12 +16,12 @@ import (
 //go:generate go run ./cmd/generate
 
 // openers contains the known datastore implementations.
-var openers map[string]func(string) ds.TxnDatastore
+var openers map[string]func(string) ds.Datastore
 
 // AddOpener allows registration of a new driver for fuzzing.
-func AddOpener(name string, opener func(loc string) ds.TxnDatastore) {
+func AddOpener(name string, opener func(loc string) ds.Datastore) {
 	if openers == nil {
-		openers = make(map[string]func(string) ds.TxnDatastore)
+		openers = make(map[string]func(string) ds.Datastore)
 	}
 	openers[name] = opener
 }
@@ -32,14 +32,16 @@ var Threads int
 
 func init() {
 	if openers == nil {
-		openers = make(map[string]func(string) ds.TxnDatastore)
+		openers = make(map[string]func(string) ds.Datastore)
 	}
 	Threads = 1
 }
 
 // RunState encapulates the state of a given fuzzing run
 type RunState struct {
-	inst          ds.TxnDatastore
+	inst ds.Datastore
+	// opMax signals which operations the instance supports.
+	opMax         op
 	inputChannels []chan<- byte
 	wg            sync.WaitGroup
 	Cancel        context.CancelFunc
@@ -50,10 +52,18 @@ type RunState struct {
 }
 
 // DB returns the datastore being driven by this instance
-func (r *RunState) DB() ds.TxnDatastore {
+func (r *RunState) DB() ds.Datastore {
 	return r.inst
 }
 
+// TxnDB returns the transaciton database if the store under test supports transactions
+func (r *RunState) TxnDB() ds.TxnDatastore {
+	if txdb, ok := r.inst.(ds.TxnDatastore); ok {
+		return txdb
+	}
+	return nil
+}
+
 type threadState struct {
 	op
 	keyReady bool
@@ -78,6 +88,11 @@ func Open(driver string, location string, cleanup bool) (*RunState, error) {
 
 	state := RunState{}
 	state.inst = opener(location)
+	state.opMax = opMax
+	// don't attempt transaction operations on non-txn datastores.
+	if state.TxnDB() == nil {
+		state.opMax = opNewTX
+	}
 	state.keyCache[0] = ds.NewKey("/")
 	state.cachedKeys = 1
 
@@ -146,7 +161,7 @@ func FuzzDB(driver string, location string, cleanup bool, data []byte) int {
 
 // FuzzStream does the same as fuzz but with streaming input
 func FuzzStream(driver string, location string, cleanup bool, data io.Reader) error {
-	inst, err := Open("badger", "tmp", true)
+	inst, err := Open(driver, location, cleanup)
 	if err != nil {
 		return err
 	}
@@ -185,10 +200,10 @@ const (
 	opQuery
 	opPut
 	opDelete
+	opSync
 	opNewTX
 	opCommitTX
 	opDiscardTX
-	opSync
 	opMax
 )
 
@@ -214,7 +229,7 @@ func threadDriver(ctx context.Context, runState *RunState, cmnds chan byte) {
 
 func nextState(s *threadState, c byte) error {
 	if s.op == opNone {
-		s.op = op(c) % opMax
+		s.op = op(c) % s.RunState.opMax
 		return nil
 	} else if s.op == opGet {
 		if !s.keyReady {
@@ -267,11 +282,13 @@ func nextState(s *threadState, c byte) error {
 		return nil
 	} else if s.op == opNewTX {
 		if s.txn == nil {
-			s.txn, _ = s.RunState.inst.NewTransaction(((c & 1) == 1))
-			if (c & 1) != 1 { // read+write
-				s.writer = s.txn
+			if tdb := s.RunState.TxnDB(); tdb != nil {
+				s.txn, _ = tdb.NewTransaction(((c & 1) == 1))
+				if (c & 1) != 1 { // read+write
+					s.writer = s.txn
+				}
+				s.reader = s.txn
 			}
-			s.reader = s.txn
 		}
 		reset(s)
 		return nil

fuzz/go.mod

@@ -6,6 +6,5 @@ replace github.com/ipfs/go-datastore => ../
 
 require (
 	github.com/ipfs/go-datastore v0.4.4
-	github.com/ipfs/go-ds-badger v0.2.4
 	github.com/spf13/pflag v1.0.3
 )

fuzz/go.sum

@@ -2,6 +2,7 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkBy
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -32,9 +33,15 @@ github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8
 github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
 github.com/ipfs/go-ds-badger v0.2.4 h1:UPGB0y7luFHk+mY/tUZrif/272M8o+hFsW+avLUeWrM=
 github.com/ipfs/go-ds-badger v0.2.4/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk=
+github.com/ipfs/go-ds-flatfs v0.4.4 h1:DmGZ4qOYQLNgu8Mltuz1DtUHpm+BjWMcVN3F3H3VJzQ=
+github.com/ipfs/go-ds-flatfs v0.4.4/go.mod h1:e4TesLyZoA8k1gV/yCuBTnt2PJtypn4XUlB5n8KQMZY=
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
 github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
+github.com/ipfs/go-log v1.0.3 h1:Gg7SUYSZ7BrqaKMwM+hRgcAkKv4QLfzP4XPQt5Sx/OI=
+github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=
+github.com/ipfs/go-log/v2 v2.0.3 h1:Q2gXcBoCALyLN/pUQlz1qgu0x3uFV6FzP9oXhpfyJpc=
+github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
 github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
 github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
 github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o=
@@ -54,6 +61,8 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg=
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -77,9 +86,15 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A=
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=