From baf03b57aec638ef8df660a5da6871ea75c20553 Mon Sep 17 00:00:00 2001 From: Karl Lessard Date: Mon, 12 Feb 2024 12:19:01 -0500 Subject: [PATCH 1/3] Exclude generation of JavaCPP bindings depending on TSL --- .github/workflows/build.yml | 2 +- .../internal/c_api/TSL_PayloadVisitor.java | 24 ------------------ .../tensorflow/internal/c_api/TSL_Status.java | 19 -------------- .../internal/c_api/global/tensorflow.java | 25 +++++-------------- .../internal/c_api/presets/tensorflow.java | 9 ++++--- 5 files changed, 13 insertions(+), 66 deletions(-) delete mode 100644 tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/TSL_PayloadVisitor.java delete mode 100644 tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/TSL_Status.java diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3971fdbf4c3..c3a4608dea5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -154,7 +154,7 @@ jobs: call mvn -version echo ^^^^ossrh^^${{ secrets.CI_DEPLOY_USERNAME }}^^${{ secrets.CI_DEPLOY_PASSWORD }}^^^^ > %USERPROFILE%\.m2\settings.xml set "SKIP_EXPORT=true" - call mvn clean install -pl "!tensorflow-framework" -Dnative.build -B -U -e -Djavacpp.platform=windows-x86_64 -Djavacpp.platform.extension=${{ matrix.ext }} -am -DstagingRepositoryId=${{ needs.prepare.outputs.stagingRepositoryId }} + call mvn clean install -pl "!tensorflow-framework" -B -U -e -Djavacpp.platform=windows-x86_64 -Djavacpp.platform.extension=${{ matrix.ext }} -am -DstagingRepositoryId=${{ needs.prepare.outputs.stagingRepositoryId }} if ERRORLEVEL 1 exit /b deploy: if: github.event_name == 'push' && contains(github.ref, 'master') diff --git a/tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/TSL_PayloadVisitor.java b/tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/TSL_PayloadVisitor.java deleted file mode 100644 index 672d781646a..00000000000 --- a/tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/TSL_PayloadVisitor.java +++ /dev/null @@ -1,24 +0,0 @@ -// Targeted by JavaCPP version 1.5.10: DO NOT EDIT THIS FILE - -package org.tensorflow.internal.c_api; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.tensorflow.internal.c_api.global.tensorflow.*; - - -// Iterates over the stored payloads and calls the `visitor(key, value)` -// callable for each one. `key` and `value` is only usable during the callback. -// `capture` will be passed to the callback without modification. -@Properties(inherit = org.tensorflow.internal.c_api.presets.tensorflow.class) -public class TSL_PayloadVisitor extends FunctionPointer { - static { Loader.load(); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TSL_PayloadVisitor(Pointer p) { super(p); } - protected TSL_PayloadVisitor() { allocate(); } - private native void allocate(); - public native void call(@Cast("const char*") BytePointer key, @Cast("const char*") BytePointer value, - Pointer capture); -} diff --git a/tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/TSL_Status.java b/tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/TSL_Status.java deleted file mode 100644 index 784e5789f92..00000000000 --- a/tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/TSL_Status.java +++ /dev/null @@ -1,19 +0,0 @@ -// Targeted by JavaCPP version 1.5.10: DO NOT EDIT THIS FILE - -package org.tensorflow.internal.c_api; - -import java.nio.*; -import org.bytedeco.javacpp.*; -import org.bytedeco.javacpp.annotation.*; - -import static org.tensorflow.internal.c_api.global.tensorflow.*; - -// #endif - -@Opaque @Properties(inherit = org.tensorflow.internal.c_api.presets.tensorflow.class) -public class TSL_Status extends Pointer { - /** Empty constructor. Calls {@code super((Pointer)null)}. */ - public TSL_Status() { super((Pointer)null); } - /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ - public TSL_Status(Pointer p) { super(p); } -} diff --git a/tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/global/tensorflow.java b/tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/global/tensorflow.java index da17905dd79..1e92da85e46 100644 --- a/tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/global/tensorflow.java +++ b/tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/global/tensorflow.java @@ -295,9 +295,7 @@ public static native void TF_TString_Copy(TF_TString dst, String src, // #define TENSORFLOW_TSL_C_TSL_STATUS_H_ // #ifdef __cplusplus -// Targeting ../TSL_Status.java - - +// #endif // -------------------------------------------------------------------------- // TSL_Code holds an error code. The enum values here are identical to @@ -325,43 +323,30 @@ public static native void TF_TString_Copy(TF_TString dst, String src, // -------------------------------------------------------------------------- // Return a new status object. -public static native TSL_Status TSL_NewStatus(); // Delete a previously created status object. -public static native void TSL_DeleteStatus(TSL_Status arg0); // Record in *s. Any previous information is lost. // A common use is to clear a status: TSL_SetStatus(s, TSL_OK, ""); -public static native void TSL_SetStatus(TSL_Status s, @Cast("TSL_Code") int code, @Cast("const char*") BytePointer msg); -public static native void TSL_SetStatus(TSL_Status s, @Cast("TSL_Code") int code, String msg); // Record as a payload in *s. The previous payload having the // same key (if any) is overwritten. Payload will not be added if the Status // is OK. -public static native void TSL_SetPayload(TSL_Status s, @Cast("const char*") BytePointer key, @Cast("const char*") BytePointer value); -public static native void TSL_SetPayload(TSL_Status s, String key, String value); -// Targeting ../TSL_PayloadVisitor.java - -public static native void TSL_ForEachPayload(@Const TSL_Status s, TSL_PayloadVisitor visitor, - Pointer capture); +// Iterates over the stored payloads and calls the `visitor(key, value)` +// callable for each one. `key` and `value` is only usable during the callback. +// `capture` will be passed to the callback without modification. // Convert from an I/O error code (e.g., errno) to a TSL_Status value. // Any previous information is lost. Prefer to use this instead of TSL_SetStatus // when the error comes from I/O operations. -public static native void TSL_SetStatusFromIOError(TSL_Status s, int error_code, - @Cast("const char*") BytePointer context); -public static native void TSL_SetStatusFromIOError(TSL_Status s, int error_code, - String context); // Return the code record in *s. -public static native @Cast("TSL_Code") int TSL_GetCode(@Const TSL_Status s); // Return a pointer to the (null-terminated) error message in *s. The // return value points to memory that is only usable until the next // mutation to *s. Always returns an empty string if TSL_GetCode(s) is // TSL_OK. -public static native @Cast("const char*") BytePointer TSL_Message(@Const TSL_Status s); // #ifdef __cplusplus /* end extern "C" */ // #endif @@ -4823,6 +4808,8 @@ public static native void TFE_InitializeLocalOnlyContext(TFE_Context ctx, // #ifndef TENSORFLOW_JAVA_SCOPE_H_ // #define TENSORFLOW_JAVA_SCOPE_H_ +// #include + // #include "tensorflow/c/c_api.h" // Targeting ../TFJ_Scope.java diff --git a/tensorflow-core/tensorflow-core-native/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java b/tensorflow-core/tensorflow-core-native/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java index 31876a34ba5..33ec1a2c930 100644 --- a/tensorflow-core/tensorflow-core-native/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java +++ b/tensorflow-core/tensorflow-core-native/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java @@ -182,10 +182,13 @@ public void map(InfoMap infoMap) { .put(new Info("TF_WhileParams") .purify()) .put(new Info("TFE_CustomDeviceTensorHandle::deallocator") - .javaNames("cdt_deallocator")) - .put(new Info("TF_PayloadVisitor", "TF_ForEachPayload") - .skip() // avoids import of TSL_PayloadVisitor + .javaNames("cdt_deallocator") ); + + // TensorFlow is remapping all TSL symbols into its own namespace, so avoid generate bindings that requires linkage + // to TSL symbols directly (at this time 02/12/2024, this is still not possible in Windows, see + // https://github.com/tensorflow/tensorflow/issues/62579) + infoMap.put(new Info("TSL_Status", "TSL_PayloadVisitor", "TF_PayloadVisitor", "TF_ForEachPayload").skip()); } @Override From 469dea587b89fe87623d58d2865f350c379776a8 Mon Sep 17 00:00:00 2001 From: Karl Lessard Date: Mon, 12 Feb 2024 12:53:23 -0500 Subject: [PATCH 2/3] Skip C++-API dependent method --- .../java/org/tensorflow/internal/c_api/global/tensorflow.java | 3 +-- .../java/org/tensorflow/internal/c_api/presets/tensorflow.java | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/global/tensorflow.java b/tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/global/tensorflow.java index 1e92da85e46..c5be5971ff1 100644 --- a/tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/global/tensorflow.java +++ b/tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/global/tensorflow.java @@ -3876,8 +3876,7 @@ public static native void TFE_ContextExportRunMetadata(TFE_Context ctx, // namespace tensorflow -public static native TFE_TensorHandle TFE_NewTensorHandle(@Const @ByRef Tensor t, - TF_Status status); + // #endif // #endif // TENSORFLOW_C_EAGER_C_API_H_ diff --git a/tensorflow-core/tensorflow-core-native/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java b/tensorflow-core/tensorflow-core-native/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java index 33ec1a2c930..e4c9b198b86 100644 --- a/tensorflow-core/tensorflow-core-native/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java +++ b/tensorflow-core/tensorflow-core-native/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java @@ -189,6 +189,9 @@ public void map(InfoMap infoMap) { // to TSL symbols directly (at this time 02/12/2024, this is still not possible in Windows, see // https://github.com/tensorflow/tensorflow/issues/62579) infoMap.put(new Info("TSL_Status", "TSL_PayloadVisitor", "TF_PayloadVisitor", "TF_ForEachPayload").skip()); + + // This C++-API dependent method appears somehow at the bottom of c/eager/c_api.h, skip it + infoMap.put(new Info("TFE_NewTensorHandle(const tensorflow::Tensor&, TF_Status*)").skip()); } @Override From 1b599829d1e909111da4ba25bea3132202276666 Mon Sep 17 00:00:00 2001 From: Karl Lessard Date: Mon, 12 Feb 2024 13:35:11 -0500 Subject: [PATCH 3/3] Use official 2.15 Windows C build --- .../scripts/dist_download.sh | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/tensorflow-core/tensorflow-core-native/scripts/dist_download.sh b/tensorflow-core/tensorflow-core-native/scripts/dist_download.sh index e9109778052..a97220538c3 100755 --- a/tensorflow-core/tensorflow-core-native/scripts/dist_download.sh +++ b/tensorflow-core/tensorflow-core-native/scripts/dist_download.sh @@ -18,8 +18,7 @@ case ${PLATFORM:-} in ;; 'windows-x86_64') WHEEL_URL='https://files.pythonhosted.org/packages/4c/48/1a5a15517f18eaa4ff8d598b1c000300b20c1bb0e624539d702117a0c369/tensorflow_intel-2.15.0-cp311-cp311-win_amd64.whl' - #CLIB_URL='https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-windows-x86_64-2.15.0.zip' - CLIB_URL='https://storage.googleapis.com/libtensorflow-nightly/prod/tensorflow/release/windows/latest/cpu/windows_cpu_libtensorflow_binaries.tar.gz' + CLIB_URL='https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-windows-x86_64-2.15.0.zip' ;; *) echo "TensorFlow distribution for ${PLATFORM} is not supported for download" @@ -39,17 +38,10 @@ fi if [[ -n "$CLIB_URL" ]]; then echo "Downloading $CLIB_URL" -# if [ ! -f 'tensorflow_c.zip' ]; then -# curl -L $CLIB_URL --output 'tensorflow_c.zip' -# fi -# yes | unzip -q -u -d tensorflow 'tensorflow_c.zip' - - # FIXME Temporary experiment to validate windows build from nightly snapshot - if [ ! -f 'tensorflow_c.tar.gz' ]; then - curl -L $CLIB_URL --output 'tensorflow_c.tar.gz' + if [ ! -f 'tensorflow_c.zip' ]; then + curl -L $CLIB_URL --output 'tensorflow_c.zip' fi - yes | tar xzvf 'tensorflow_c.tar.gz' - yes | unzip -q -u -d tensorflow 'lib_package/libtensorflow-cpu-windows-x86_64.zip' + yes | unzip -q -u -d tensorflow 'tensorflow_c.zip' fi cd tensorflow