From 9f13d63154b588a31fccb024b336dbe4466722c9 Mon Sep 17 00:00:00 2001 From: Rong Ou Date: Thu, 20 May 2021 13:11:12 -0700 Subject: [PATCH 1/2] use address and length for GDS reads/writes --- java/src/main/java/ai/rapids/cudf/CuFile.java | 53 +++++++++++++++++-- .../test/java/ai/rapids/cudf/CuFileTest.java | 12 ++--- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/CuFile.java b/java/src/main/java/ai/rapids/cudf/CuFile.java index 00c9cdb9fd5..5b321feef67 100644 --- a/java/src/main/java/ai/rapids/cudf/CuFile.java +++ b/java/src/main/java/ai/rapids/cudf/CuFile.java @@ -78,11 +78,26 @@ public static boolean libraryLoaded() { * @param path The file path to copy to. * @param file_offset The file offset from which to write the buffer. * @param buffer The device buffer to copy from. - * @return The file offset from which the buffer was appended. + * @deprecated Use writeDeviceMemoryToFile instead. */ public static void writeDeviceBufferToFile(File path, long file_offset, BaseDeviceMemoryBuffer buffer) { - writeToFile(path.getAbsolutePath(), file_offset, buffer.getAddress(), buffer.getLength()); + writeDeviceMemoryToFile(path, file_offset, buffer.getAddress(), buffer.getLength()); + } + + /** + * Write device memory to a given file path synchronously. + *

+ * This method is NOT thread safe if the path points to the same file on disk. + * + * @param path The file path to copy to. + * @param file_offset The file offset from which to write the buffer. + * @param address The device memory address to copy from. + * @param length The length to copy. + */ + public static void writeDeviceMemoryToFile(File path, long file_offset, long address, + long length) { + writeToFile(path.getAbsolutePath(), file_offset, address, length); } /** @@ -93,9 +108,24 @@ public static void writeDeviceBufferToFile(File path, long file_offset, * @param path The file path to copy to. * @param buffer The device buffer to copy from. * @return The file offset from which the buffer was appended. + * @deprecated Use appendDeviceMemoryToFile instead. */ public static long appendDeviceBufferToFile(File path, BaseDeviceMemoryBuffer buffer) { - return appendToFile(path.getAbsolutePath(), buffer.getAddress(), buffer.getLength()); + return appendDeviceMemoryToFile(path, buffer.getAddress(), buffer.getLength()); + } + + /** + * Append device memory to a given file path synchronously. + *

+ * This method is NOT thread safe if the path points to the same file on disk. + * + * @param path The file path to copy to. + * @param address The device memory address to copy from. + * @param length The length to copy. + * @return The file offset from which the buffer was appended. + */ + public static long appendDeviceMemoryToFile(File path, long address, long length) { + return appendToFile(path.getAbsolutePath(), address, length); } /** @@ -106,10 +136,25 @@ public static long appendDeviceBufferToFile(File path, BaseDeviceMemoryBuffer bu * @param buffer The device buffer to copy into. * @param path The file path to copy from. * @param fileOffset The file offset from which to copy the content. + * @deprecated Use readFileToDeviceMemory instead. */ public static void readFileToDeviceBuffer(BaseDeviceMemoryBuffer buffer, File path, long fileOffset) { - readFromFile(buffer.getAddress(), buffer.getLength(), path.getAbsolutePath(), fileOffset); + readFileToDeviceMemory(buffer.getAddress(), buffer.getLength(), path, fileOffset); + } + + /** + * Read a file into device memory synchronously. + *

+ * This method is NOT thread safe if the path points to the same file on disk. + * + * @param address The device memory address to read into. + * @param length The length to read. + * @param path The file path to copy from. + * @param fileOffset The file offset from which to copy the content. + */ + public static void readFileToDeviceMemory(long address, long length, File path, long fileOffset) { + readFromFile(address, length, path.getAbsolutePath(), fileOffset); } private static native void writeToFile(String path, long file_offset, long address, long length); diff --git a/java/src/test/java/ai/rapids/cudf/CuFileTest.java b/java/src/test/java/ai/rapids/cudf/CuFileTest.java index 10415cae893..72b4d1d8279 100644 --- a/java/src/test/java/ai/rapids/cudf/CuFileTest.java +++ b/java/src/test/java/ai/rapids/cudf/CuFileTest.java @@ -73,8 +73,8 @@ private void verifyCopyToFile(File tempFile) { HostMemoryBuffer dest = HostMemoryBuffer.allocate(16)) { orig.setLong(0, 123456789); from.copyFromHostBuffer(orig); - CuFile.writeDeviceBufferToFile(tempFile, 0, from); - CuFile.readFileToDeviceBuffer(to, tempFile, 0); + CuFile.writeDeviceMemoryToFile(tempFile, 0, from.address, from.length); + CuFile.readFileToDeviceMemory(to.address, to.length, tempFile, 0); dest.copyFromDeviceBuffer(to); assertEquals(123456789, dest.getLong(0)); } @@ -87,17 +87,17 @@ private void verifyAppendToFile(File tempFile) { HostMemoryBuffer dest = HostMemoryBuffer.allocate(16)) { orig.setLong(0, 123456789); from.copyFromHostBuffer(orig); - assertEquals(0, CuFile.appendDeviceBufferToFile(tempFile, from)); + assertEquals(0, CuFile.appendDeviceMemoryToFile(tempFile, from.address, from.length)); orig.setLong(0, 987654321); from.copyFromHostBuffer(orig); - assertEquals(16, CuFile.appendDeviceBufferToFile(tempFile, from)); + assertEquals(16, CuFile.appendDeviceMemoryToFile(tempFile, from.address, from.length)); - CuFile.readFileToDeviceBuffer(to, tempFile, 0); + CuFile.readFileToDeviceMemory(to.address, to.length, tempFile, 0); dest.copyFromDeviceBuffer(to); assertEquals(123456789, dest.getLong(0)); - CuFile.readFileToDeviceBuffer(to, tempFile, 16); + CuFile.readFileToDeviceMemory(to.address, to.length, tempFile, 16); dest.copyFromDeviceBuffer(to); assertEquals(987654321, dest.getLong(0)); } From ca5f1441867f52e04cdd1ed640d8424a98c5a636 Mon Sep 17 00:00:00 2001 From: Rong Ou Date: Thu, 20 May 2021 14:54:26 -0700 Subject: [PATCH 2/2] un-deprecate buffer methods --- java/src/main/java/ai/rapids/cudf/CuFile.java | 3 --- java/src/test/java/ai/rapids/cudf/CuFileTest.java | 12 ++++++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/CuFile.java b/java/src/main/java/ai/rapids/cudf/CuFile.java index 5b321feef67..4baad834570 100644 --- a/java/src/main/java/ai/rapids/cudf/CuFile.java +++ b/java/src/main/java/ai/rapids/cudf/CuFile.java @@ -78,7 +78,6 @@ public static boolean libraryLoaded() { * @param path The file path to copy to. * @param file_offset The file offset from which to write the buffer. * @param buffer The device buffer to copy from. - * @deprecated Use writeDeviceMemoryToFile instead. */ public static void writeDeviceBufferToFile(File path, long file_offset, BaseDeviceMemoryBuffer buffer) { @@ -108,7 +107,6 @@ public static void writeDeviceMemoryToFile(File path, long file_offset, long add * @param path The file path to copy to. * @param buffer The device buffer to copy from. * @return The file offset from which the buffer was appended. - * @deprecated Use appendDeviceMemoryToFile instead. */ public static long appendDeviceBufferToFile(File path, BaseDeviceMemoryBuffer buffer) { return appendDeviceMemoryToFile(path, buffer.getAddress(), buffer.getLength()); @@ -136,7 +134,6 @@ public static long appendDeviceMemoryToFile(File path, long address, long length * @param buffer The device buffer to copy into. * @param path The file path to copy from. * @param fileOffset The file offset from which to copy the content. - * @deprecated Use readFileToDeviceMemory instead. */ public static void readFileToDeviceBuffer(BaseDeviceMemoryBuffer buffer, File path, long fileOffset) { diff --git a/java/src/test/java/ai/rapids/cudf/CuFileTest.java b/java/src/test/java/ai/rapids/cudf/CuFileTest.java index 72b4d1d8279..10415cae893 100644 --- a/java/src/test/java/ai/rapids/cudf/CuFileTest.java +++ b/java/src/test/java/ai/rapids/cudf/CuFileTest.java @@ -73,8 +73,8 @@ private void verifyCopyToFile(File tempFile) { HostMemoryBuffer dest = HostMemoryBuffer.allocate(16)) { orig.setLong(0, 123456789); from.copyFromHostBuffer(orig); - CuFile.writeDeviceMemoryToFile(tempFile, 0, from.address, from.length); - CuFile.readFileToDeviceMemory(to.address, to.length, tempFile, 0); + CuFile.writeDeviceBufferToFile(tempFile, 0, from); + CuFile.readFileToDeviceBuffer(to, tempFile, 0); dest.copyFromDeviceBuffer(to); assertEquals(123456789, dest.getLong(0)); } @@ -87,17 +87,17 @@ private void verifyAppendToFile(File tempFile) { HostMemoryBuffer dest = HostMemoryBuffer.allocate(16)) { orig.setLong(0, 123456789); from.copyFromHostBuffer(orig); - assertEquals(0, CuFile.appendDeviceMemoryToFile(tempFile, from.address, from.length)); + assertEquals(0, CuFile.appendDeviceBufferToFile(tempFile, from)); orig.setLong(0, 987654321); from.copyFromHostBuffer(orig); - assertEquals(16, CuFile.appendDeviceMemoryToFile(tempFile, from.address, from.length)); + assertEquals(16, CuFile.appendDeviceBufferToFile(tempFile, from)); - CuFile.readFileToDeviceMemory(to.address, to.length, tempFile, 0); + CuFile.readFileToDeviceBuffer(to, tempFile, 0); dest.copyFromDeviceBuffer(to); assertEquals(123456789, dest.getLong(0)); - CuFile.readFileToDeviceMemory(to.address, to.length, tempFile, 16); + CuFile.readFileToDeviceBuffer(to, tempFile, 16); dest.copyFromDeviceBuffer(to); assertEquals(987654321, dest.getLong(0)); }