rocksdb

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ concurrency:
 jobs:
   lint:
     name: Swift Lint
-    runs-on: [self-hosted, linux]
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v4
@@ -34,7 +34,11 @@ jobs:
     - name: Get blst submodule commit hash
       id: blst-commit-hash
       run: |
-        echo "::set-output name=commit-hash::$(git submodule status Utils/Sources/blst/ | cut -c2-41)"
+        echo "commit-hash=$(git submodule status Utils/Sources/blst/ | cut -c2-41)" >> $GITHUB_OUTPUT
+    - name: Get rocksdb submodule commit hash
+      id: rocksdb-commit-hash
+      run: |
+        echo "commit-hash=$(git submodule status Database/Sources/rocksdb/ | cut -c2-41)" >> $GITHUB_OUTPUT
     - name: Cache SPM
       uses: actions/cache@v4
       with:
@@ -66,6 +70,13 @@ jobs:
         key: ${{ runner.os }}-libs-libbandersnatch-${{ hashFiles('Utils/Sources/bandersnatch/**') }}
         restore-keys: |
             ${{ runner.os }}-libs-libbandersnatch
+    - name: Cache rocksdb static lib
+      uses: actions/cache@v4
+      with:
+        path: .lib/librocksdb.a
+        key: ${{ runner.os }}-libs-librocksdb-${{ steps.rocksdb-commit-hash.outputs.commit-hash }}
+        restore-keys: |
+            ${{ runner.os }}-libs-librocksdb
     - name: Setup Swift
       uses: SwiftyLab/setup-swift@latest
       with:

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "blst"]
 	path = Utils/Sources/blst
 	url = https://github.com/supranational/blst.git
+[submodule "Database/Sources/rocksdb"]
+	path = Database/Sources/rocksdb
+	url = https://github.com/facebook/rocksdb.git

.swiftlint.yml

@@ -6,6 +6,7 @@ disabled_rules:
     - nesting
     - opening_brace
     - cyclomatic_complexity
+    - blanket_disable_command
 
 excluded:
     - "**/.build"

Database/Package.resolved

@@ -0,0 +1,24 @@
+{
+  "originHash" : "45b033b1ce14d3e50b4dcbf706d87b3f7431c36dc9d513397f7be8a04999d298",
+  "pins" : [
+    {
+      "identity" : "swift-syntax",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-syntax.git",
+      "state" : {
+        "revision" : "4c6cc0a3b9e8f14b3ae2307c5ccae4de6167ac2c",
+        "version" : "600.0.0-prerelease-2024-06-12"
+      }
+    },
+    {
+      "identity" : "swift-testing",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-testing.git",
+      "state" : {
+        "branch" : "0.10.0",
+        "revision" : "69d59cfc76e5daf498ca61f5af409f594768eef9"
+      }
+    }
+  ],
+  "version" : 3
+}

Database/Package.swift

@@ -5,22 +5,43 @@ import PackageDescription
 
 let package = Package(
     name: "Database",
+    platforms: [
+        .macOS(.v14),
+    ],
     products: [
         // Products define the executables and libraries a package produces, making them visible to other packages.
         .library(
             name: "Database",
             targets: ["Database"]
         ),
     ],
+    dependencies: [
+        .package(url: "https://github.com/apple/swift-testing.git", branch: "0.10.0"),
+    ],
     targets: [
         // Targets are the basic building blocks of a package, defining a module or a test suite.
         // Targets can depend on other targets in this package and products from dependencies.
         .target(
-            name: "Database"
+            name: "Database",
+            dependencies: [
+                "rocksdb",
+            ],
+            linkerSettings: [
+                .unsafeFlags(["-L../.lib"]),
+                .linkedLibrary("z"),
+                .linkedLibrary("bz2"),
+            ]
+        ),
+        .systemLibrary(
+            name: "rocksdb",
+            path: "Sources"
         ),
         .testTarget(
             name: "DatabaseTests",
-            dependencies: ["Database"]
+            dependencies: [
+                "Database",
+                .product(name: "Testing", package: "swift-testing"),
+            ]
         ),
     ],
     swiftLanguageVersions: [.version("6")]

Database/Sources/Database/RocksDB.swift

@@ -0,0 +1,190 @@
+import Foundation
+import rocksdb
+
+public final class RocksDB {
+    public enum BatchOperation {
+        case delete(key: Data)
+        case put(key: Data, value: Data)
+    }
+
+    public enum Error: Swift.Error {
+        case openFailed(message: String)
+        case putFailed(message: String)
+        case getFailed(message: String)
+        case deleteFailed(message: String)
+        case batchFailed(message: String)
+        case noData
+    }
+
+    private let dbOptions: OpaquePointer
+    private let writeOptions: OpaquePointer
+    private let readOptions: OpaquePointer
+    private let db: OpaquePointer
+
+    public init(path: URL) throws(Error) {
+        let dbOptions = rocksdb_options_create()
+        self.dbOptions = dbOptions!
+        let cpus = sysconf(Int32(_SC_NPROCESSORS_ONLN))
+
+        // Optimize rocksdb
+        rocksdb_options_increase_parallelism(dbOptions, Int32(cpus))
+        rocksdb_options_optimize_level_style_compaction(dbOptions, 0) // TODO: check this
+
+        // create the DB if it's not already present
+        rocksdb_options_set_create_if_missing(dbOptions, 1)
+
+        // create writeoptions
+        writeOptions = rocksdb_writeoptions_create()
+        // create readoptions
+        readOptions = rocksdb_readoptions_create()
+
+        // open DB
+        db = try Self.call { err, _ in
+            rocksdb_open(dbOptions, path.path, &err)
+        } onErr: { message throws(Error) in
+            throw Error.openFailed(message: message)
+        }
+    }
+
+    deinit {
+        rocksdb_writeoptions_destroy(writeOptions)
+        rocksdb_readoptions_destroy(readOptions)
+        rocksdb_options_destroy(dbOptions)
+        rocksdb_close(db)
+    }
+}
+
+// MARK: - private helpers
+
+extension RocksDB {
+    private static func call<R>(
+        _ data: [Data],
+        fn: (inout UnsafeMutablePointer<Int8>?, [(ptr: UnsafeRawPointer, count: Int)]) -> R,
+        // need new swiftlint version https://github.com/realm/SwiftLint/issues/5631
+        // swiftlint:disable:next identifier_name
+        onErr: (String) throws(Error) -> Void
+    ) throws(Error) -> R {
+        var err: UnsafeMutablePointer<Int8>?
+        defer {
+            free(err)
+        }
+
+        func helper(data: ArraySlice<Data>, ptr: [(ptr: UnsafeRawPointer, count: Int)]) -> Result<
+            R, Error
+        > {
+            if data.isEmpty {
+                return .success(fn(&err, ptr))
+            }
+            let rest = data.dropFirst()
+            let first = data.first!
+            return first.withUnsafeBytes {
+                (bufferPtr: UnsafeRawBufferPointer) -> Result<R, Error> in
+                guard let bufferAddress = bufferPtr.baseAddress else {
+                    return .failure(.noData)
+                }
+                return helper(data: rest, ptr: ptr + [(bufferAddress, bufferPtr.count)])
+            }
+        }
+
+        let ret = helper(data: data[...], ptr: [])
+
+        switch ret {
+        case let .success(value):
+            if let pointee = err {
+                let message = String(cString: pointee)
+                try onErr(message)
+            }
+            return value
+        case let .failure(error):
+            throw error
+        }
+    }
+
+    private static func call<R>(
+        _ data: Data...,
+        fn: (inout UnsafeMutablePointer<Int8>?, [(ptr: UnsafeRawPointer, count: Int)]) -> R,
+        onErr: (String) throws(Error) -> Void
+    ) throws(Error) -> R {
+        try call(data, fn: fn, onErr: onErr)
+    }
+
+    private static func call<R>(
+        _ data: Data...,
+        fn: ([(ptr: UnsafeRawPointer, count: Int)]) -> R
+    ) throws(Error) -> R {
+        try call(data) { _, ptrs in
+            fn(ptrs)
+        } onErr: { _ throws(Error) in
+            // do nothing as it should never be called
+        }
+    }
+}
+
+// MARK: - public methods
+
+extension RocksDB {
+    public func put(key: Data, value: Data) throws(Error) {
+        try Self.call(key, value) { err, ptrs in
+            let key = ptrs[0]
+            let value = ptrs[1]
+            rocksdb_put(db, writeOptions, key.ptr, key.count, value.ptr, value.count, &err)
+        } onErr: { message throws(Error) in
+            throw Error.putFailed(message: message)
+        }
+    }
+
+    public func get(key: Data) throws -> Data? {
+        var len = 0
+
+        let ret = try Self.call(key) { err, ptrs in
+            let key = ptrs[0]
+            return rocksdb_get(db, readOptions, key.ptr, key.count, &len, &err)
+        } onErr: { message throws(Error) in
+            throw Error.getFailed(message: message)
+        }
+
+        defer {
+            free(ret)
+        }
+
+        return ret.map { Data(bytes: $0, count: len) }
+    }
+
+    public func delete(key: Data) throws {
+        try Self.call(key) { err, ptrs in
+            let key = ptrs[0]
+            rocksdb_delete(db, writeOptions, key.ptr, key.count, &err)
+        } onErr: { message throws(Error) in
+            throw Error.deleteFailed(message: message)
+        }
+    }
+
+    public func batch(operations: [BatchOperation]) throws {
+        let writeBatch = rocksdb_writebatch_create()
+        defer { rocksdb_writebatch_destroy(writeBatch) }
+
+        for operation in operations {
+            switch operation {
+            case let .delete(key):
+                try Self.call(key) { ptrs in
+                    let key = ptrs[0]
+                    rocksdb_writebatch_delete(writeBatch, key.ptr, key.count)
+                }
+
+            case let .put(key, value):
+                try Self.call(key, value) { ptrs in
+                    let key = ptrs[0]
+                    let value = ptrs[1]
+
+                    rocksdb_writebatch_put(writeBatch, key.ptr, key.count, value.ptr, value.count)
+                }
+            }
+        }
+
+        try Self.call { err, _ in
+            rocksdb_write(db, writeOptions, writeBatch, &err)
+        } onErr: { message throws(Error) in
+            throw Error.batchFailed(message: message)
+        }
+    }
+}

Database/Sources/module.modulemap

@@ -0,0 +1,4 @@
+module rocksdb {
+  header "rocksdb/include/rocksdb/c.h"
+  link "rocksdb"
+}

Database/Tests/DatabaseTests/RocksDBTests.swift

@@ -0,0 +1,64 @@
+// swiftlint:disable force_try
+import Foundation
+import Testing
+
+@testable import Database
+
+extension String {
+    var data: Data {
+        Data(utf8)
+    }
+}
+
+final class RocksDBTests {
+    let path = {
+        let tmpDir = FileManager.default.temporaryDirectory
+        return tmpDir.appendingPathComponent("\(UUID().uuidString)")
+    }()
+
+    var rocksDB: RocksDB!
+
+    init() throws {
+        rocksDB = try RocksDB(path: path)
+    }
+
+    deinit {
+        rocksDB = nil // close it first
+        // then delete the files
+        try! FileManager.default.removeItem(at: path)
+    }
+
+    @Test func basicOperations() throws {
+        #expect(try rocksDB.get(key: "123".data) == nil)
+
+        try rocksDB.put(key: "123".data, value: "qwe".data)
+        try rocksDB.put(key: "234".data, value: "asd".data)
+
+        #expect(try rocksDB.get(key: "123".data) == "qwe".data)
+        #expect(try rocksDB.get(key: "234".data) == "asd".data)
+
+        try rocksDB.delete(key: "123".data)
+
+        #expect(try rocksDB.get(key: "123".data) == nil)
+
+        try rocksDB.put(key: "234".data, value: "asdfg".data)
+
+        #expect(try rocksDB.get(key: "234".data) == "asdfg".data)
+    }
+
+    @Test func testBatchOperations() throws {
+        try rocksDB.put(key: "123".data, value: "qwe".data)
+
+        try rocksDB.batch(operations: [
+            .delete(key: "123".data),
+            .put(key: "234".data, value: "wer".data),
+            .put(key: "345".data, value: "ert".data),
+            .delete(key: "234".data),
+            .put(key: "345".data, value: "ertert".data),
+        ])
+
+        #expect(try rocksDB.get(key: "123".data) == nil)
+        #expect(try rocksDB.get(key: "234".data) == nil)
+        #expect(try rocksDB.get(key: "345".data) == "ertert".data)
+    }
+}