From 1c97a490dda8e8b8d3805fa6debccc34f32b540d Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Thu, 12 Jun 2014 09:23:30 -0600 Subject: [PATCH] Add Windows support. This commit adds Windows support to Bolt. Windows memory maps return an address instead of a byte slice so the DB.data field had to be refactored to be a pointer to a large byte array. --- bolt_386.go | 4 ++++ bolt_amd64.go | 4 ++++ bolt_darwin.go | 32 ++++++++++++++++++++++++++------ bolt_linux.go | 32 ++++++++++++++++++++++++++------ bolt_windows.go | 37 +++++++++++++++++++++++++------------ db.go | 24 +++++++++--------------- doc.go | 5 +++-- 7 files changed, 97 insertions(+), 41 deletions(-) create mode 100644 bolt_386.go create mode 100644 bolt_amd64.go diff --git a/bolt_386.go b/bolt_386.go new file mode 100644 index 000000000..856f40119 --- /dev/null +++ b/bolt_386.go @@ -0,0 +1,4 @@ +package bolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0xFFFFFFF // 256MB diff --git a/bolt_amd64.go b/bolt_amd64.go new file mode 100644 index 000000000..4262932ee --- /dev/null +++ b/bolt_amd64.go @@ -0,0 +1,4 @@ +package bolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/bolt_darwin.go b/bolt_darwin.go index 557de72fa..2bffad394 100644 --- a/bolt_darwin.go +++ b/bolt_darwin.go @@ -3,6 +3,7 @@ package bolt import ( "os" "syscall" + "unsafe" ) var odirect int @@ -22,12 +23,31 @@ func funlock(f *os.File) error { return syscall.Flock(int(f.Fd()), syscall.LOCK_UN) } -// mmap memory maps a file to a byte slice. -func mmap(f *os.File, sz int) ([]byte, error) { - return syscall.Mmap(int(f.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED) +// mmap memory maps a DB's data file. +func mmap(db *DB, sz int) error { + b, err := syscall.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED) + if err != nil { + return err + } + + // Save the original byte slice and convert to a byte array pointer. + db.dataref = b + db.data = (*[maxMapSize]byte)(unsafe.Pointer(&b[0])) + db.datasz = sz + return nil } -// munmap unmaps a pointer from a file. -func munmap(b []byte) error { - return syscall.Munmap(b) +// munmap unmaps a DB's data file from memory. +func munmap(db *DB) error { + // Ignore the unmap if we have no mapped data. + if db.dataref == nil { + return nil + } + + // Unmap using the original byte slice. + err := syscall.Munmap(db.dataref) + db.dataref = nil + db.data = nil + db.datasz = 0 + return err } diff --git a/bolt_linux.go b/bolt_linux.go index 9aba95587..761a83edb 100644 --- a/bolt_linux.go +++ b/bolt_linux.go @@ -3,6 +3,7 @@ package bolt import ( "os" "syscall" + "unsafe" ) var odirect = syscall.O_DIRECT @@ -22,12 +23,31 @@ func funlock(f *os.File) error { return syscall.Flock(int(f.Fd()), syscall.LOCK_UN) } -// mmap memory maps a file to a byte slice. -func mmap(f *os.File, sz int) ([]byte, error) { - return syscall.Mmap(int(f.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED) +// mmap memory maps a DB's data file. +func mmap(db *DB, sz int) error { + b, err := syscall.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED) + if err != nil { + return err + } + + // Save the original byte slice and convert to a byte array pointer. + db.dataref = b + db.data = (*[maxMapSize]byte)(unsafe.Pointer(&b[0])) + db.datasz = sz + return nil } -// munmap unmaps a pointer from a file. -func munmap(b []byte) error { - return syscall.Munmap(b) +// munmap unmaps a DB's data file from memory. +func munmap(db *DB) error { + // Ignore the unmap if we have no mapped data. + if db.dataref == nil { + return nil + } + + // Unmap using the original byte slice. + err := syscall.Munmap(db.dataref) + db.dataref = nil + db.data = nil + db.datasz = 0 + return err } diff --git a/bolt_windows.go b/bolt_windows.go index 9962f03aa..8300d40a4 100644 --- a/bolt_windows.go +++ b/bolt_windows.go @@ -1,6 +1,7 @@ package bolt import ( + "fmt" "os" "syscall" "unsafe" @@ -23,36 +24,48 @@ func funlock(f *os.File) error { return nil } -// mmap memory maps a file to a byte slice. +// mmap memory maps a DB's data file. // Based on: https://github.com/edsrzf/mmap-go -func mmap(f *os.File, sz int) ([]byte, error) { +func mmap(db *DB, sz int) error { + // Truncate the database to the size of the mmap. + if err := db.file.Truncate(int64(sz)); err != nil { + return fmt.Errorf("truncate: %s", err) + } + // Open a file mapping handle. - sizelo, sizehi := uint32(sz>>32), uint32(sz&0xffffffff) - h, errno := syscall.CreateFileMapping(syscall.Handle(f.Fd()), nil, syscall.PAGE_READONLY, sizehi, sizelo, nil) + sizelo := uint32(sz >> 32) + sizehi := uint32(sz & 0xffffffff) + h, errno := syscall.CreateFileMapping(syscall.Handle(db.file.Fd()), nil, syscall.PAGE_READONLY, sizelo, sizehi, nil) if h == 0 { - return nil, os.NewSyscallError("CreateFileMapping", errno) + return os.NewSyscallError("CreateFileMapping", errno) } // Create the memory map. addr, errno := syscall.MapViewOfFile(h, syscall.FILE_MAP_READ, 0, 0, uintptr(sz)) if addr == 0 { - return nil, os.NewSyscallError("MapViewOfFile", errno) + return os.NewSyscallError("MapViewOfFile", errno) } // Close mapping handle. if err := syscall.CloseHandle(syscall.Handle(h)); err != nil { - return nil, os.NewSyscallError("CloseHandle", err) + return os.NewSyscallError("CloseHandle", err) } - // Convert to a byte slice. - b := ((*[0xFFFFFFF]byte)(unsafe.Pointer(addr)))[0:sz] - return b, nil + // Convert to a byte array. + db.data = ((*[maxMapSize]byte)(unsafe.Pointer(addr))) + db.datasz = sz + + return nil } // munmap unmaps a pointer from a file. // Based on: https://github.com/edsrzf/mmap-go -func munmap(b []byte) error { - addr := (uintptr)(unsafe.Pointer(&b[0])) +func munmap(db *DB) error { + if db.data == nil { + return nil + } + + addr := (uintptr)(unsafe.Pointer(&db.data[0])) if err := syscall.UnmapViewOfFile(addr); err != nil { return os.NewSyscallError("UnmapViewOfFile", err) } diff --git a/db.go b/db.go index 2132f9c24..8f8051563 100644 --- a/db.go +++ b/db.go @@ -67,7 +67,9 @@ type DB struct { path string file *os.File - data []byte + dataref []byte + data *[maxMapSize]byte + datasz int meta0 *meta meta1 *meta pageSize int @@ -191,13 +193,8 @@ func (db *DB) mmap(minsz int) error { } size = db.mmapSize(size) - // Truncate the database to the size of the mmap. - if err := db.file.Truncate(int64(size)); err != nil { - return fmt.Errorf("truncate: %s", err) - } - // Memory-map the data file as a byte slice. - if db.data, err = mmap(db.file, size); err != nil { + if err := mmap(db, size); err != nil { return err } @@ -218,11 +215,8 @@ func (db *DB) mmap(minsz int) error { // munmap unmaps the data file from memory. func (db *DB) munmap() error { - if db.data != nil { - if err := munmap(db.data); err != nil { - return fmt.Errorf("unmap error: " + err.Error()) - } - db.data = nil + if err := munmap(db); err != nil { + return fmt.Errorf("unmap error: " + err.Error()) } return nil } @@ -507,7 +501,7 @@ func (db *DB) Stats() Stats { // This is for internal access to the raw data bytes from the C cursor, use // carefully, or not at all. func (db *DB) Info() *Info { - return &Info{db.data, db.pageSize} + return &Info{uintptr(unsafe.Pointer(&db.data[0])), db.pageSize} } // page retrieves a page reference from the mmap based on the current page size. @@ -544,7 +538,7 @@ func (db *DB) allocate(count int) (*page, error) { // Resize mmap() if we're at the end. p.id = db.rwtx.meta.pgid var minsz = int((p.id+pgid(count))+1) * db.pageSize - if minsz >= len(db.data) { + if minsz >= db.datasz { if err := db.mmap(minsz); err != nil { return nil, fmt.Errorf("mmap allocate error: %s", err) } @@ -579,7 +573,7 @@ func (s *Stats) add(other *Stats) { } type Info struct { - Data []byte + Data uintptr PageSize int } diff --git a/doc.go b/doc.go index caf66e9d9..b127f594e 100644 --- a/doc.go +++ b/doc.go @@ -12,6 +12,9 @@ rolled back in the event of a crash. The design of Bolt is based on Howard Chu's LMDB database project. +Bolt currently works on Windows, Mac OS X, and Linux. + + Basics There are only a few types in Bolt: DB, Bucket, Tx, and Cursor. The DB is @@ -33,7 +36,5 @@ values returned from Bolt cannot be changed. Writing to a read-only byte slice will cause Go to panic. If you need to work with data returned from a Get() you need to first copy it to a new byte slice. -Bolt currently works on Mac OS and Linux. Windows support is coming soon. - */ package bolt