Integrate jlibdeflate for faster DEFLATE compression and decompression

build.gradle

@@ -16,6 +16,11 @@ plugins {
 
 repositories {
     mavenCentral()
+    maven {
+        name = 'sonatypeSnapshots'
+        url = 'https://central.sonatype.com/repository/maven-snapshots/'
+        mavenContent { snapshotsOnly() }
+    }
 }
 
 jacocoTestReport {
@@ -29,6 +34,7 @@ jacocoTestReport {
 }
 
 dependencies {
+    implementation 'com.fulcrumgenomics:jlibdeflate:0.1.0'
     implementation 'commons-logging:commons-logging:1.3.0'
     implementation "org.xerial.snappy:snappy-java:1.1.10.5"
     implementation 'org.apache.commons:commons-compress:1.26.0'

src/main/java/htsjdk/samtools/Defaults.java

@@ -103,6 +103,13 @@ public class Defaults {
     public static final boolean SRA_LIBRARIES_DOWNLOAD;
 
 
+    /**
+     * Whether to attempt to use jlibdeflate (libdeflate via JNI) for DEFLATE compression and decompression.
+     * When true, the default deflater/inflater factories will try to load the native library and fall back
+     * to the JDK implementation if it is not available.  Default = true.
+     */
+    public static final boolean USE_LIBDEFLATE;
+
     /**
      * The name of the system property that disables snappy.  Default = "snappy.disable".
      */
@@ -138,6 +145,7 @@ public class Defaults {
         CUSTOM_READER_FACTORY = getStringProperty("custom_reader", "");
         SAM_FLAG_FIELD_FORMAT = SamFlagField.valueOf(getStringProperty("sam_flag_field_format", SamFlagField.DECIMAL.name()));
         SRA_LIBRARIES_DOWNLOAD = getBooleanProperty("sra_libraries_download", false);
+        USE_LIBDEFLATE = getBooleanProperty("use_libdeflate", true);
         DISABLE_SNAPPY_COMPRESSOR = getBooleanProperty(DISABLE_SNAPPY_PROPERTY_NAME, false);
         OPTIMISTIC_VCF_4_4 = getBooleanProperty(OPTIMISTIC_VCF_4_4_PROPERTY, false);
     }
@@ -162,6 +170,7 @@ public static SortedMap<String, Object> allDefaults(){
         result.put("EBI_REFERENCE_SERVICE_URL_MASK", EBI_REFERENCE_SERVICE_URL_MASK);
         result.put("CUSTOM_READER_FACTORY", CUSTOM_READER_FACTORY);
         result.put("SAM_FLAG_FIELD_FORMAT", SAM_FLAG_FIELD_FORMAT);
+        result.put("USE_LIBDEFLATE", USE_LIBDEFLATE);
         result.put("DISABLE_SNAPPY_COMPRESSOR", DISABLE_SNAPPY_COMPRESSOR);
         return Collections.unmodifiableSortedMap(result);
     }

src/main/java/htsjdk/samtools/util/zip/DeflaterFactory.java

@@ -23,14 +23,25 @@
  */
 package htsjdk.samtools.util.zip;
 
+import htsjdk.samtools.Defaults;
 import htsjdk.samtools.util.BlockCompressedOutputStream;
+import htsjdk.samtools.util.Log;
+
 import java.util.zip.Deflater;
 
 /**
  * Factory for {@link Deflater} objects used by {@link BlockCompressedOutputStream}.
  * This class may be extended to provide alternative deflaters (e.g., for improved performance).
+ *
+ * <p>By default, if {@link Defaults#USE_LIBDEFLATE} is true, this factory will attempt to
+ * create a {@link LibdeflateDeflater} backed by the libdeflate native library.  If the native
+ * library is not available, it falls back to the JDK {@link Deflater}.</p>
  */
 public class DeflaterFactory {
+    private static final Log log = Log.getInstance(DeflaterFactory.class);
+
+    /** Cached result of whether libdeflate is available; null means not yet tested. */
+    private static volatile Boolean libdeflateAvailable;
 
     public DeflaterFactory() {
         //Note: made explicit constructor to make searching for references easier
@@ -43,6 +54,39 @@ public DeflaterFactory() {
      * @param gzipCompatible if true then use GZIP compatible compression
      */
     public Deflater makeDeflater(final int compressionLevel, final boolean gzipCompatible) {
+        if (Defaults.USE_LIBDEFLATE && isLibdeflateAvailable()) {
+            return new LibdeflateDeflater(compressionLevel, gzipCompatible);
+        }
         return new Deflater(compressionLevel, gzipCompatible);
     }
+
+    /** Returns true if the libdeflate native library can be loaded. */
+    static boolean isLibdeflateAvailable() {
+        if (libdeflateAvailable == null) {
+            synchronized (DeflaterFactory.class) {
+                if (libdeflateAvailable == null) {
+                    libdeflateAvailable = testLibdeflate();
+                }
+            }
+        }
+        return libdeflateAvailable;
+    }
+
+    private static boolean testLibdeflate() {
+        try {
+            final LibdeflateDeflater deflater = new LibdeflateDeflater(1, true);
+            try {
+                deflater.setInput(new byte[]{0}, 0, 1);
+                deflater.finish();
+                deflater.deflate(new byte[16], 0, 16);
+            } finally {
+                deflater.end();
+            }
+            log.info("libdeflate is available; using libdeflate for DEFLATE compression.");
+            return true;
+        } catch (final Throwable t) {
+            log.info(t, "libdeflate is not available; falling back to JDK deflater.");
+            return false;
+        }
+    }
 }

src/main/java/htsjdk/samtools/util/zip/InflaterFactory.java

@@ -23,13 +23,18 @@
  */
 package htsjdk.samtools.util.zip;
 
+import htsjdk.samtools.Defaults;
 import htsjdk.samtools.util.BlockGunzipper;
+
 import java.util.zip.Inflater;
 
 /**
  * Factory for {@link Inflater} objects used by {@link BlockGunzipper}.
  * This class may be extended to provide alternative inflaters (e.g., for improved performance).
- * The default implementation returns a JDK {@link Inflater}
+ *
+ * <p>By default, if {@link Defaults#USE_LIBDEFLATE} is true and the native library is available,
+ * this factory will create a {@link LibdeflateInflater}.  Otherwise it falls back to the
+ * JDK {@link Inflater}.</p>
  */
 public class InflaterFactory {
 
@@ -40,10 +45,12 @@ public InflaterFactory() {
     /**
      * Returns an inflater object that will be used when reading DEFLATE compressed files.
      * Subclasses may override to provide their own inflater implementation.
-     * The default implementation returns a JDK {@link Inflater}
      * @param gzipCompatible if true then use GZIP compatible compression
      */
     public Inflater makeInflater(final boolean gzipCompatible) {
+        if (Defaults.USE_LIBDEFLATE && DeflaterFactory.isLibdeflateAvailable()) {
+            return new LibdeflateInflater(gzipCompatible);
+        }
         return new Inflater(gzipCompatible);
     }
 }

src/main/java/htsjdk/samtools/util/zip/LibdeflateDeflater.java

@@ -0,0 +1,116 @@
+/*
+ * The MIT License
+ *
+ * Copyright (c) 2026 Tim Fennell
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package htsjdk.samtools.util.zip;
+
+import com.fulcrumgenomics.jlibdeflate.LibdeflateCompressor;
+
+import java.util.zip.Deflater;
+
+/**
+ * A {@link Deflater} implementation backed by libdeflate via the jlibdeflate library.
+ * Provides significantly faster DEFLATE compression than the JDK's built-in zlib.
+ *
+ * <p>This class supports the subset of the Deflater API used by BGZF block compression:
+ * {@link #reset()}, {@link #setInput(byte[], int, int)}, {@link #finish()},
+ * {@link #deflate(byte[], int, int)}, and {@link #finished()}.</p>
+ */
+class LibdeflateDeflater extends Deflater {
+
+    private final LibdeflateCompressor compressor;
+    private final boolean nowrap;
+
+    private byte[] inputBuf;
+    private int inputOff;
+    private int inputLen;
+    private boolean finishing;
+    private boolean done;
+
+    /**
+     * Creates a new LibdeflateDeflater at the specified compression level.
+     *
+     * @param level   compression level (0-12 for libdeflate, 0-9 for standard compatibility)
+     * @param nowrap  if true, produce raw DEFLATE (no zlib/gzip header); if false, produce zlib format
+     */
+    LibdeflateDeflater(final int level, final boolean nowrap) {
+        // The super constructor allocates a native zlib stream we won't use.
+        // We immediately free it since all compression goes through libdeflate.
+        super(level, nowrap);
+        super.end();
+
+        this.nowrap = nowrap;
+        this.compressor = new LibdeflateCompressor(level);
+    }
+
+    @Override
+    public void setInput(final byte[] input, final int off, final int len) {
+        this.inputBuf = input;
+        this.inputOff = off;
+        this.inputLen = len;
+        this.done = false;
+    }
+
+    @Override
+    public void finish() {
+        this.finishing = true;
+    }
+
+    @Override
+    public int deflate(final byte[] output, final int off, final int len) {
+        if (inputBuf == null || inputLen == 0) {
+            done = true;
+            return 0;
+        }
+
+        final int compressed = nowrap
+                ? compressor.deflateCompress(inputBuf, inputOff, inputLen, output, off, len)
+                : compressor.zlibCompress(inputBuf, inputOff, inputLen, output, off, len);
+        if (compressed == -1) {
+            // Output buffer too small — caller will handle this (e.g. fall back to no-compression)
+            done = false;
+            return 0;
+        }
+
+        done = true;
+        return compressed;
+    }
+
+    @Override
+    public boolean finished() {
+        return finishing && done;
+    }
+
+    @Override
+    public void reset() {
+        inputBuf  = null;
+        inputOff  = 0;
+        inputLen  = 0;
+        finishing = false;
+        done      = false;
+    }
+
+    @Override
+    public void end() {
+        compressor.close();
+    }
+}

src/main/java/htsjdk/samtools/util/zip/LibdeflateInflater.java

@@ -0,0 +1,98 @@
+/*
+ * The MIT License
+ *
+ * Copyright (c) 2026 Tim Fennell
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package htsjdk.samtools.util.zip;
+
+import com.fulcrumgenomics.jlibdeflate.LibdeflateDecompressor;
+
+import java.util.zip.DataFormatException;
+import java.util.zip.Inflater;
+
+/**
+ * An {@link Inflater} implementation backed by libdeflate via the jlibdeflate library.
+ * Provides significantly faster DEFLATE decompression than the JDK's built-in zlib.
+ *
+ * <p>This class supports the subset of the Inflater API used by BGZF block decompression:
+ * {@link #reset()}, {@link #setInput(byte[], int, int)}, and
+ * {@link #inflate(byte[], int, int)}.</p>
+ *
+ * <p>The libdeflate decompressor requires the exact uncompressed size to be known. In BGZF,
+ * this is always the case since the uncompressed size is stored in the block footer and
+ * passed as the {@code len} parameter to {@link #inflate(byte[], int, int)}.</p>
+ */
+class LibdeflateInflater extends Inflater {
+
+    private final LibdeflateDecompressor decompressor;
+    private final boolean nowrap;
+
+    private byte[] inputBuf;
+    private int inputOff;
+    private int inputLen;
+
+    LibdeflateInflater(final boolean nowrap) {
+        // The super constructor allocates a native zlib stream we won't use.
+        // We immediately free it since all decompression goes through libdeflate.
+        super(nowrap);
+        super.end();
+
+        this.nowrap = nowrap;
+        this.decompressor = new LibdeflateDecompressor();
+    }
+
+    @Override
+    public void setInput(final byte[] input, final int off, final int len) {
+        this.inputBuf = input;
+        this.inputOff = off;
+        this.inputLen = len;
+    }
+
+    @Override
+    public int inflate(final byte[] output, final int off, final int len) throws DataFormatException {
+        if (inputBuf == null || inputLen == 0 || len == 0) {
+            return 0;
+        }
+
+        try {
+            if (nowrap) {
+                decompressor.deflateDecompress(inputBuf, inputOff, inputLen, output, off, len);
+            } else {
+                decompressor.zlibDecompress(inputBuf, inputOff, inputLen, output, off, len);
+            }
+            return len;
+        } catch (final Exception e) {
+            throw new DataFormatException(e.getMessage());
+        }
+    }
+
+    @Override
+    public void reset() {
+        inputBuf = null;
+        inputOff = 0;
+        inputLen = 0;
+    }
+
+    @Override
+    public void end() {
+        decompressor.close();
+    }
+}