Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make Snappy technically an optional dependency #1715

Merged
merged 1 commit into from
Sep 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 22 additions & 27 deletions src/main/java/htsjdk/samtools/util/SnappyLoader.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,25 +25,22 @@

import htsjdk.samtools.Defaults;
import htsjdk.samtools.SAMException;
import org.xerial.snappy.SnappyError;
import org.xerial.snappy.SnappyInputStream;
import org.xerial.snappy.SnappyOutputStream;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

/**
* Checks if Snappy is available, and provides methods for wrapping InputStreams and OutputStreams with Snappy if so.
* Checks if Snappy is available, and provides methods for wrapping InputStreams and OutputStreams with Snappy if it is.
*
* @implNote this class must not import Snappy code in order to prevent exceptions if the Snappy Library is not available.
* Snappy code is handled by {@link SnappyLoaderInternal}.
*/
public class SnappyLoader {
private static final int SNAPPY_BLOCK_SIZE = 32768; // keep this as small as can be without hurting compression ratio.
private static final Log logger = Log.getInstance(SnappyLoader.class);

private final boolean snappyAvailable;


public SnappyLoader() {
this(Defaults.DISABLE_SNAPPY_COMPRESSOR);
}
Expand All @@ -52,24 +49,18 @@ public SnappyLoader() {
if (disableSnappy) {
logger.debug("Snappy is disabled via system property.");
snappyAvailable = false;
}
else {
boolean tmpSnappyAvailable = false;
try (final OutputStream test = new SnappyOutputStream(new ByteArrayOutputStream(1000))){
test.write("Hello World!".getBytes());
tmpSnappyAvailable = true;
logger.debug("Snappy successfully loaded.");
}
/*
* ExceptionInInitializerError: thrown by Snappy if native libs fail to load.
* IllegalStateException: thrown within the `test.write` call above if no UTF-8 encoder is found.
* IOException: potentially thrown by the `test.write` and `test.close` calls.
* SnappyError: potentially thrown for a variety of reasons by Snappy.
*/
catch (final ExceptionInInitializerError | IllegalStateException | IOException | SnappyError e) {
logger.warn(e, "Snappy native library failed to load.");
} else {
boolean tmpAvailable;
try {
//This triggers trying to import Snappy code, which causes an exception if the library is missing.
tmpAvailable = SnappyLoaderInternal.tryToLoadSnappy();
} catch (NoClassDefFoundError e){
tmpAvailable = false;
logger.error(e, "Snappy java library was requested but not found. If Snappy is " +
"intentionally missing, this message may be suppressed by setting " +
"-D"+ Defaults.SAMJDK_PREFIX + Defaults.DISABLE_SNAPPY_PROPERTY_NAME + "=true " );
}
snappyAvailable = tmpSnappyAvailable;
snappyAvailable = tmpAvailable;
}
}

Expand All @@ -81,18 +72,21 @@ public SnappyLoader() {
* @throws SAMException if Snappy is not available will throw an exception.
*/
public InputStream wrapInputStream(final InputStream inputStream) {
return wrapWithSnappyOrThrow(inputStream, SnappyInputStream::new);
return wrapWithSnappyOrThrow(inputStream, SnappyLoaderInternal.getInputStreamWrapper());
}

/**
* Wrap an OutputStream in a SnappyOutputStream.
* @throws SAMException if Snappy is not available
*/
public OutputStream wrapOutputStream(final OutputStream outputStream) {
return wrapWithSnappyOrThrow(outputStream, (stream) -> new SnappyOutputStream(stream, SNAPPY_BLOCK_SIZE));
return wrapWithSnappyOrThrow(outputStream, SnappyLoaderInternal.getOutputStreamWrapper());
}

private interface IOFunction<T,R> {
/**
* Function which can throw IOExceptions
*/
interface IOFunction<T,R> {
R apply(T input) throws IOException;
}

Expand All @@ -111,4 +105,5 @@ private <T,R> R wrapWithSnappyOrThrow(T stream, IOFunction<T, R> wrapper){
throw new SAMException(errorMessage);
}
}

}
69 changes: 69 additions & 0 deletions src/main/java/htsjdk/samtools/util/SnappyLoaderInternal.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package htsjdk.samtools.util;

import htsjdk.annotations.InternalAPI;
import org.xerial.snappy.SnappyError;
import org.xerial.snappy.SnappyInputStream;
import org.xerial.snappy.SnappyOutputStream;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

/**
* This class is the only one which should actually import Snappy Classes. It is separated from SnappyLoader to allow
* snappy to be an optional dependency. Referencing snappy classes directly if the library is unavailable causes a
* NoClassDefFoundError, so use this instead.
*
* This should only be referenced by {@link SnappyLoader} in order to prevent accidental imports of Snappy classes.
*
*/
@InternalAPI
class SnappyLoaderInternal {
private static final Log logger = Log.getInstance(SnappyLoaderInternal.class);
private static final int SNAPPY_BLOCK_SIZE = 32768; // keep this as small as can be without hurting compression ratio.

/**
* Try to load Snappy's native library.
*
* Note that calling this when snappy is not available will throw NoClassDefFoundError!
*
* @return true iff Snappy's native libraries are loaded and functioning.
*/
static boolean tryToLoadSnappy() {
final boolean snappyAvailable;
boolean tmpSnappyAvailable = false;
try (final OutputStream test = new SnappyOutputStream(new ByteArrayOutputStream(1000))){
test.write("Hello World!".getBytes());
tmpSnappyAvailable = true;
logger.debug("Snappy successfully loaded.");
}
/*
* ExceptionInInitializerError: thrown by Snappy if native libs fail to load.
* IllegalStateException: thrown within the `test.write` call above if no UTF-8 encoder is found.
* IOException: potentially thrown by the `test.write` and `test.close` calls.
* SnappyError: potentially thrown for a variety of reasons by Snappy.
*/
catch (final ExceptionInInitializerError | IllegalStateException | IOException | SnappyError e) {
logger.warn(e, "Snappy native library failed to load.");
}
snappyAvailable = tmpSnappyAvailable;
return snappyAvailable;
}


/**
* @return a function which wraps an InputStream in a new SnappyInputStream
*/
static SnappyLoader.IOFunction<InputStream, InputStream> getInputStreamWrapper(){
return SnappyInputStream::new;
}

/**
* @return a function which wraps an OutputStream in a new SnappyOutputStream with an appropriate block size
*/
static SnappyLoader.IOFunction<OutputStream, OutputStream> getOutputStreamWrapper(){
return (stream) -> new SnappyOutputStream(stream, SNAPPY_BLOCK_SIZE);
}

}
Loading