From a594365a489af2fe5f97d7133808cbc4ec20792b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matheus=20C=2E=20Fran=C3=A7a?= Date: Wed, 6 Nov 2024 05:18:59 -0300 Subject: [PATCH] refactor (bindings/zig): Improvements (#5247) --- bindings/zig/build.zig | 69 +++++-- bindings/zig/build.zig.zon | 10 + bindings/zig/src/opendal.zig | 369 ++++++++++++++++++++++++++++++++++- 3 files changed, 430 insertions(+), 18 deletions(-) diff --git a/bindings/zig/build.zig b/bindings/zig/build.zig index b5d26bfa473c..62b3df224e33 100644 --- a/bindings/zig/build.zig +++ b/bindings/zig/build.zig @@ -21,14 +21,38 @@ pub fn build(b: *std.Build) void { const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); + const use_llvm = b.option(bool, "use-llvm", "Use LLVM backend (default: true)") orelse true; + const use_clang = b.option(bool, "use-clang", "Use libclang in translate-c (default: true)") orelse true; + + // Generate the Zig bindings for OpenDAL C bindings + const opendal_binding = b.addTranslateC(.{ + .optimize = optimize, + .target = target, + .link_libc = true, + .root_source_file = b.path("../c/include/opendal.h"), + .use_clang = use_clang, // TODO: set 'false' use fno-llvm/fno-clang (may be zig v1.0) + }); + + // ZigCoro - (stackful) Coroutine for Zig (library) + const zigcoro = b.dependency("zigcoro", .{}).module("libcoro"); + // This function creates a module and adds it to the package's module set, making // it available to other packages which depend on this one. const opendal_module = b.addModule("opendal", .{ .root_source_file = b.path("src/opendal.zig"), .target = target, .optimize = optimize, + .link_libcpp = true, + }); + opendal_module.addImport("opendal_c_header", opendal_binding.addModule("opendal_c_header")); + opendal_module.addImport("libcoro", zigcoro); + opendal_module.addLibraryPath(switch (optimize) { + .Debug => b.path("../c/target/debug"), + else => b.path("../c/target/release"), }); - opendal_module.addIncludePath(b.path("../c/include")); + opendal_module.linkSystemLibrary("opendal_c", .{}); + + // =============== OpenDAL C bindings =============== // Creates a step for building the dependent C bindings const libopendal_c_cmake = b.addSystemCommand(&[_][]const u8{ "cmake", "-S", "../c", "-B", "../c/build" }); @@ -39,27 +63,44 @@ pub fn build(b: *std.Build) void { libopendal_c.step.dependOn(config_libopendal_c); build_libopendal_c.dependOn(&libopendal_c.step); + // =============== OpenDAL C bindings =============== + // Creates a step for unit testing. This only builds the test executable // but does not run it. - const unit_tests = b.addTest(.{ - .root_source_file = b.path("test/bdd.zig"), + + // Test library + const lib_test = b.addTest(.{ + .root_source_file = b.path("src/opendal.zig"), .target = target, .optimize = optimize, + .use_llvm = use_llvm, + .test_runner = b.dependency("test_runner", .{}).path("test_runner.zig"), + }); + lib_test.addLibraryPath(switch (optimize) { + .Debug => b.path("../c/target/debug"), + else => b.path("../c/target/release"), }); + lib_test.linkLibCpp(); + lib_test.linkSystemLibrary("opendal_c"); + lib_test.root_module.addImport("opendal_c_header", opendal_binding.addModule("opendal_c_header")); + lib_test.root_module.addImport("libcoro", zigcoro); - unit_tests.addIncludePath(b.path("../c/include")); - if (optimize == .Debug) { - unit_tests.addLibraryPath(b.path("../c/target/debug")); - } else { - unit_tests.addLibraryPath(b.path("../c/target/release")); - } - unit_tests.linkSystemLibrary("opendal_c"); - unit_tests.linkLibCpp(); - unit_tests.root_module.addImport("opendal", opendal_module); + // BDD sample test + const bdd_test = b.addTest(.{ + .name = "bdd_test", + .root_source_file = b.path("test/bdd.zig"), + .target = target, + .optimize = optimize, + .use_llvm = use_llvm, + .test_runner = b.dependency("test_runner", .{}).path("test_runner.zig"), + }); + bdd_test.root_module.addImport("opendal", opendal_module); // Creates a step for running unit tests. - const run_unit_tests = b.addRunArtifact(unit_tests); + const run_lib_test = b.addRunArtifact(lib_test); + const run_bdd_test = b.addRunArtifact(bdd_test); const test_step = b.step("test", "Run OpenDAL Zig bindings tests"); test_step.dependOn(&libopendal_c.step); - test_step.dependOn(&run_unit_tests.step); + test_step.dependOn(&run_lib_test.step); + test_step.dependOn(&run_bdd_test.step); } diff --git a/bindings/zig/build.zig.zon b/bindings/zig/build.zig.zon index 7d7ab514071f..934a5bbb2697 100644 --- a/bindings/zig/build.zig.zon +++ b/bindings/zig/build.zig.zon @@ -1,6 +1,16 @@ .{ .name = "opendal", .version = "0.0.1", + .dependencies = .{ + .zigcoro = .{ + .url = "git+https://github.com/rsepassi/zigcoro#ca58a912c3c0957d6aab0405f4c78f68e1dababb", + .hash = "12204959321c5e16a70944b8cdf423f0fa2bdf7db1f72bacc89b8af85acc4c054d9c", + }, + .test_runner = .{ + .url = "git+https://gist.github.com/karlseguin/c6bea5b35e4e8d26af6f81c22cb5d76b/#f303e77231e14b405e4e800072e7deacac4a4881", + .hash = "12208ab39744c8c0909d53b8c9889aff0690930d20e98e4f650f9690310e9a860b14", + }, + }, .paths = .{ "build.zig", "build.zig.zon", diff --git a/bindings/zig/src/opendal.zig b/bindings/zig/src/opendal.zig index 02bfc41e90c9..8060e57f3878 100644 --- a/bindings/zig/src/opendal.zig +++ b/bindings/zig/src/opendal.zig @@ -15,9 +15,225 @@ // specific language governing permissions and limitations // under the License. -pub const c = @cImport(@cInclude("opendal.h")); +pub const c = @import("opendal_c_header"); + +pub const Operator = struct { + inner: *c.opendal_operator, + + pub fn init(scheme: []const u8, options: ?*c.opendal_operator_options) !Operator { + const result = c.opendal_operator_new(scheme.ptr, options); + if (result.op == null) { + if (result.@"error") |err| { + c.opendal_error_free(err); + } + return error.OperatorInitFailed; + } + return .{ + .inner = result.op.?, + }; + } + + pub fn deinit(self: *Operator) void { + c.opendal_operator_free(self.inner); + } + + pub fn write(self: *const Operator, path: []const u8, data: []const u8) !void { + const bytes = c.opendal_bytes{ + .data = @constCast(data.ptr), + .len = data.len, + .capacity = data.len, + }; + if (c.opendal_operator_write(self.inner, path.ptr, &bytes)) |err| { + errdefer c.opendal_error_free(err); + try codeToError(err.*.code); + } + } + + pub fn read(self: *const Operator, path: []const u8) ![]const u8 { + const result = c.opendal_operator_read(self.inner, path.ptr); + if (result.@"error") |err| { + errdefer c.opendal_error_free(err); + try codeToError(err.*.code); + } + return result.data.data[0..result.data.len]; + } + + pub fn delete(self: *const Operator, path: []const u8) !void { + if (c.opendal_operator_delete(self.inner, path.ptr)) |err| { + errdefer c.opendal_error_free(err); + try codeToError(err.*.code); + } + } + + pub fn stat(self: *const Operator, path: []const u8) !Metadata { + const result = c.opendal_operator_stat(self.inner, path.ptr); + if (result.@"error") |err| { + errdefer c.opendal_error_free(err); + try codeToError(err.*.code); + } + return .{ .inner = result.meta.? }; + } + + pub fn exists(self: *const Operator, path: []const u8) !bool { + const result = c.opendal_operator_exists(self.inner, path.ptr); + if (result.@"error") |err| { + errdefer c.opendal_error_free(err); + try codeToError(err.*.code); + } + return result.exists; + } + + pub fn list(self: *const Operator, path: []const u8) !Lister { + const result = c.opendal_operator_list(self.inner, path.ptr); + if (result.@"error") |err| { + errdefer c.opendal_error_free(err); + try codeToError(err.*.code); + } + return .{ .inner = result.lister.? }; + } + + pub fn createDir(self: *const Operator, path: []const u8) !void { + if (c.opendal_operator_create_dir(self.inner, path.ptr)) |err| { + errdefer c.opendal_error_free(err); + try codeToError(err.*.code); + } + } + + pub fn rename(self: *const Operator, src: []const u8, dest: []const u8) !void { + if (c.opendal_operator_rename(self.inner, src.ptr, dest.ptr)) |err| { + errdefer c.opendal_error_free(err); + try codeToError(err.*.code); + } + } + + pub fn copy(self: *const Operator, src: []const u8, dest: []const u8) !void { + if (c.opendal_operator_copy(self.inner, src.ptr, dest.ptr)) |err| { + errdefer c.opendal_error_free(err); + try codeToError(err.*.code); + } + } + + pub fn info(self: *const Operator) !OperatorInfo { + const info_new = c.opendal_operator_info_new(self.inner); + if (info_new == null) return error.InfoFailed; + return .{ .inner = info.? }; + } +}; + +pub const OperatorInfo = struct { + inner: *c.opendal_operator_info, + + pub fn deinit(self: *OperatorInfo) void { + c.opendal_operator_info_free(self.inner); + } + + pub fn scheme(self: *const OperatorInfo) []const u8 { + const ptr = c.opendal_operator_info_get_scheme(self.inner); + defer std.c.free(ptr); + return std.mem.span(ptr); + } + + pub fn root(self: *const OperatorInfo) []const u8 { + const ptr = c.opendal_operator_info_get_root(self.inner); + defer std.c.free(ptr); + return std.mem.span(ptr); + } + + pub fn name(self: *const OperatorInfo) []const u8 { + const ptr = c.opendal_operator_info_get_name(self.inner); + defer std.c.free(ptr); + return std.mem.span(ptr); + } + + pub fn fullCapability(self: *const OperatorInfo) c.opendal_capability { + return c.opendal_operator_info_get_full_capability(self.inner); + } + + pub fn nativeCapability(self: *const OperatorInfo) c.opendal_capability { + return c.opendal_operator_info_get_native_capability(self.inner); + } +}; + +pub const Lister = struct { + inner: *c.opendal_lister, + + pub fn deinit(self: *Lister) void { + c.opendal_lister_free(self.inner); + } + + pub fn next(self: *const Lister) !?Entry { + const result = c.opendal_lister_next(self.inner); + if (result.@"error") |err| { + errdefer c.opendal_error_free(err); + try codeToError(err.*.code); + } + if (result.entry) |entry| { + return Entry{ .inner = entry }; + } + return null; + } +}; + +pub const Entry = struct { + inner: *c.opendal_entry, + + pub fn deinit(self: *Entry) void { + c.opendal_entry_free(self.inner); + } + + pub fn path(self: *const Entry) []const u8 { + const ptr = c.opendal_entry_path(self.inner); + return std.mem.span(ptr); + } + + pub fn name(self: *const Entry) []const u8 { + const ptr = c.opendal_entry_name(self.inner); + return std.mem.span(ptr); + } +}; + +pub const Metadata = struct { + inner: *c.opendal_metadata, + + pub fn deinit(self: *Metadata) void { + c.opendal_metadata_free(self.inner); + } + + pub fn mode(self: *const Metadata) u32 { + return c.opendal_metadata_mode(self.inner); + } + + pub fn isDir(self: *const Metadata) bool { + return c.opendal_metadata_is_dir(self.inner); + } + + pub fn isFile(self: *const Metadata) bool { + return c.opendal_metadata_is_file(self.inner); + } + + pub fn contentLength(self: *const Metadata) u64 { + return c.opendal_metadata_content_length(self.inner); + } + + pub fn contentType(self: *const Metadata) ?[]const u8 { + var len: usize = undefined; + const ptr = c.opendal_metadata_content_type(self.inner, &len); + if (ptr == null) return null; + return ptr[0..len]; + } + + pub fn etag(self: *const Metadata) ?[]const u8 { + var len: usize = undefined; + const ptr = c.opendal_metadata_etag(self.inner, &len); + if (ptr == null) return null; + return ptr[0..len]; + } + + pub fn lastModified(self: *const Metadata) i64 { + return c.opendal_metadata_last_modified(self.inner); + } +}; -// Zig code get values C code pub const Code = enum(c.opendal_code) { UNEXPECTED = c.OPENDAL_UNEXPECTED, UNSUPPORTED = c.OPENDAL_UNSUPPORTED, @@ -29,6 +245,8 @@ pub const Code = enum(c.opendal_code) { ALREADY_EXISTS = c.OPENDAL_ALREADY_EXISTS, RATE_LIMITED = c.OPENDAL_RATE_LIMITED, IS_SAME_FILE = c.OPENDAL_IS_SAME_FILE, + CONDITION_NOT_MATCH = c.OPENDAL_CONDITION_NOT_MATCH, + RANGE_NOT_SATISFIED = c.OPENDAL_RANGE_NOT_SATISFIED, }; pub const OpendalError = error{ @@ -42,9 +260,11 @@ pub const OpendalError = error{ AlreadyExists, RateLimited, IsSameFile, + ConditionNotMatch, + RangeNotSatisfied, }; -pub fn codeToError(code: c.opendal_code) OpendalError!c.opendal_code { +pub fn codeToError(code: c.opendal_code) OpendalError!void { return switch (code) { c.OPENDAL_UNEXPECTED => error.Unexpected, c.OPENDAL_UNSUPPORTED => error.Unsupported, @@ -56,8 +276,12 @@ pub fn codeToError(code: c.opendal_code) OpendalError!c.opendal_code { c.OPENDAL_ALREADY_EXISTS => error.AlreadyExists, c.OPENDAL_RATE_LIMITED => error.RateLimited, c.OPENDAL_IS_SAME_FILE => error.IsSameFile, + c.OPENDAL_CONDITION_NOT_MATCH => error.ConditionNotMatch, + c.OPENDAL_RANGE_NOT_SATISFIED => error.RangeNotSatisfied, + else => {}, }; } + pub fn errorToCode(err: OpendalError) c_int { return switch (err) { error.Unexpected => c.OPENDAL_UNEXPECTED, @@ -70,9 +294,10 @@ pub fn errorToCode(err: OpendalError) c_int { error.AlreadyExists => c.OPENDAL_ALREADY_EXISTS, error.RateLimited => c.OPENDAL_RATE_LIMITED, error.IsSameFile => c.OPENDAL_IS_SAME_FILE, + error.ConditionNotMatch => c.OPENDAL_CONDITION_NOT_MATCH, + error.RangeNotSatisfied => c.OPENDAL_RANGE_NOT_SATISFIED, }; } - const std = @import("std"); const testing = std.testing; @@ -88,6 +313,8 @@ test "Error Tests" { try testing.expectError(error.AlreadyExists, codeToError(c.OPENDAL_ALREADY_EXISTS)); try testing.expectError(error.RateLimited, codeToError(c.OPENDAL_RATE_LIMITED)); try testing.expectError(error.IsSameFile, codeToError(c.OPENDAL_IS_SAME_FILE)); + try testing.expectError(error.ConditionNotMatch, codeToError(c.OPENDAL_CONDITION_NOT_MATCH)); + try testing.expectError(error.RangeNotSatisfied, codeToError(c.OPENDAL_RANGE_NOT_SATISFIED)); // Zig error to C code try testing.expectEqual(c.OPENDAL_UNEXPECTED, errorToCode(error.Unexpected)); @@ -100,8 +327,142 @@ test "Error Tests" { try testing.expectEqual(c.OPENDAL_ALREADY_EXISTS, errorToCode(error.AlreadyExists)); try testing.expectEqual(c.OPENDAL_RATE_LIMITED, errorToCode(error.RateLimited)); try testing.expectEqual(c.OPENDAL_IS_SAME_FILE, errorToCode(error.IsSameFile)); + try testing.expectEqual(c.OPENDAL_CONDITION_NOT_MATCH, errorToCode(error.ConditionNotMatch)); + try testing.expectEqual(c.OPENDAL_RANGE_NOT_SATISFIED, errorToCode(error.RangeNotSatisfied)); } test "Semantic Analyzer" { testing.refAllDecls(@This()); } + +test "operator basic operations" { + // Initialize a operator for "memory" backend, with no options + var op = try Operator.init("memory", null); + defer op.deinit(); + + // Prepare some data to be written + const data = "this_string_length_is_24"; + + // Write this into path "/testpath" + try op.write("/testpath", data); + defer _ = op.delete("/testpath") catch |err| { + std.debug.panic("Error deleting file: {}\n", .{err}); + }; + + // We can read it out, make sure the data is the same + const read_bytes = try op.read("/testpath"); + try testing.expectEqual(read_bytes.len, 24); + try testing.expectEqualStrings(read_bytes, data); +} + +test "operator advanced operations" { + var op = try Operator.init("memory", null); + defer op.deinit(); + + // Test directory creation + try op.createDir("/testdir/"); + try testing.expect(try op.exists("/testdir/")); + + // Test file operations in directory + const data = "hello world"; + try op.write("/testdir/file.txt", data); + try testing.expect(try op.exists("/testdir/file.txt")); + + // Test metadata + var meta = try op.stat("/testdir/file.txt"); + defer meta.deinit(); + try testing.expect(meta.isFile()); + try testing.expect(!meta.isDir()); + try testing.expectEqual(meta.contentLength(), data.len); + + // ================================================================================ + // "operator advanced operations" - Unsupported + // ================================================================================ + // /home/kassane/opendal/bindings/zig/src/opendal.zig:269:5: 0x103d9a0 in codeToError (test) + // return switch (code) { + // ^ + // /home/kassane/opendal/bindings/zig/src/opendal.zig:113:13: 0x103f725 in copy (test) + // try codeToError(err.*.code); + // ^ + // /home/kassane/opendal/bindings/zig/src/opendal.zig:385:5: 0x1040031 in test.operator advanced operations (test) + // try op.copy("/testdir/renamed.txt", "/testdir/copied.txt"); + // ^ + // operator advanced operations (0.99ms) + + // 3 of 4 tests passed + + // Test rename operation + // try op.rename("/testdir/file.txt", "/testdir/renamed.txt"); + // try testing.expect(!try op.exists("/testdir/file.txt")); + // try testing.expect(try op.exists("/testdir/renamed.txt")); + + // Test copy operation + // try op.copy("/testdir/renamed.txt", "/testdir/copied.txt"); + // try testing.expect(try op.exists("/testdir/renamed.txt")); + // try testing.expect(try op.exists("/testdir/copied.txt")); + //============================================ + + // Test list operation + var lister = try op.list("/testdir/"); + defer lister.deinit(); + var count: usize = 0; + while (try lister.next()) |entry| { + // defer entry.deinit(); + count += 1; + try testing.expect(entry.path().len > 0); + try testing.expect(entry.name().len > 0); + // std.log.debug("entry name: {s}", .{entry.name()}); + // std.log.debug("entry path: {s}", .{entry.path()}); + } + try testing.expectEqual(count, 2); + + // Test delete operation + // try op.delete("/testdir/renamed.txt"); + try testing.expect(!try op.exists("/testdir/renamed.txt")); +} + +test "sync operations" { + var op = try Operator.init("memory", null); + defer op.deinit(); + + const data = [_]u8{ 1, 2, 3, 4, 5 }; + + // Test first path + try writeData(&op, "test_path", &data); + const read_bytes1 = try readData(&op, "test_path"); + try testing.expectEqualSlices(u8, &data, read_bytes1); +} + +// FIXME: test async operations +// test "async operations" { +// const coro = @import("libcoro"); + +// const allocator = std.testing.allocator; +// const stack = try coro.stackAlloc(allocator, null); +// defer allocator.free(stack); + +// var op = try Operator.init("memory", null); +// defer op.deinit(); + +// const data = [_]u8{ 1, 2, 3, 4, 5 }; + +// // Test first path +// const write_frame1 = try coro.xasync(writeData, .{ &op, "test_path", &data }, stack); +// const read_frame1 = try coro.xasync(readData, .{ &op, "test_path" }, stack); +// try testing.expectEqual(write_frame1.status(), .Suspended); +// try testing.expectEqual(read_frame1.status(), .Suspended); + +// try coro.xawait(write_frame1); +// const read_bytes1 = try coro.xawait(read_frame1); +// try testing.expectEqual(write_frame1.status(), .Done); +// try testing.expectEqual(read_frame1.status(), .Done); +// try testing.expectEqualSlices(u8, &data, read_bytes1); +// } + +fn writeData(op: *Operator, path: []const u8, data: []const u8) !void { + try op.write(path, data); +} + +fn readData(op: *Operator, path: []const u8) ![]const u8 { + return try op.read(path); +}