Merge pull request #1380 from tdrwenski/fix-checksum-error

Fix invalid checksum error
Unidata · Aug 23, 2024 · 42186c7 · 42186c7
2 parents 1dac294 + b320279
commit 42186c7
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 17 deletions.
diff --git a/cdm/core/src/main/java/ucar/nc2/filter/Shuffle.java b/cdm/core/src/main/java/ucar/nc2/filter/Shuffle.java
@@ -39,7 +39,7 @@ public int getId() {
 
   @Override
   public byte[] encode(byte[] dataIn) {
-    if (dataIn.length % elemSize != 0 || elemSize <= 1) {
+    if (elemSize <= 1) {
       return dataIn;
     }
 
@@ -56,29 +56,35 @@ public byte[] encode(byte[] dataIn) {
       }
     }
 
+    int leftoverBytes = dataIn.length % this.elemSize;
+    System.arraycopy(dataIn, dataIn.length - leftoverBytes, result, result.length - leftoverBytes, leftoverBytes);
+
     return result;
   }
 
   @Override
   public byte[] decode(byte[] dataIn) {
-    if (dataIn.length % this.elemSize == 0 && this.elemSize > 1) {
-      int nElems = dataIn.length / this.elemSize;
-      byte[] result = new byte[dataIn.length];
-
-      for (int j = 0; j < this.elemSize; ++j) {
-        int sourceIndex = j * nElems;
-        int destIndex = j;
-        for (int i = 0; i < nElems; ++i) {
-          result[destIndex] = dataIn[sourceIndex];
-          sourceIndex++;
-          destIndex += this.elemSize;
-        }
-      }
-
-      return result;
-    } else {
+    if (elemSize <= 1) {
       return dataIn;
     }
+
+    int nElems = dataIn.length / this.elemSize;
+    byte[] result = new byte[dataIn.length];
+
+    for (int j = 0; j < this.elemSize; ++j) {
+      int sourceIndex = j * nElems;
+      int destIndex = j;
+      for (int i = 0; i < nElems; ++i) {
+        result[destIndex] = dataIn[sourceIndex];
+        sourceIndex++;
+        destIndex += this.elemSize;
+      }
+    }
+
+    int leftoverBytes = dataIn.length % this.elemSize;
+    System.arraycopy(dataIn, dataIn.length - leftoverBytes, result, result.length - leftoverBytes, leftoverBytes);
+
+    return result;
   }
 
   public static class Provider implements FilterProvider {

diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java b/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java
@@ -11,6 +11,7 @@
 import ucar.unidata.io.RandomAccessFile;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
@@ -59,6 +60,27 @@ public void testShuffle() throws IOException {
     testEncodeDecode(filter, "shuffle");
   }
 
+  @Test
+  public void shouldShuffleDoublesAndLeaveIntAtEnd() throws IOException {
+    ByteBuffer bb = ByteBuffer.allocate(2 * 8 + 4);
+    bb.putDouble(-1.0);
+    bb.putDouble(1.0);
+    // to represent a checksum at the end of a chunk of doubles
+    bb.putInt(12345);
+    byte[] bytes = bb.array();
+
+    Map<String, Object> props = new HashMap<>();
+    props.put("id", "shuffle");
+    props.put("elementsize", 8);
+    Filter filter = new Shuffle(props);
+
+    byte[] expected = new byte[] {-65, 63, -16, -16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 57};
+    byte[] encoded = filter.encode(bytes);
+    assertThat(encoded).isEqualTo(expected);
+    byte[] decoded = filter.decode(encoded);
+    assertThat(decoded).isEqualTo(bytes);
+  }
+
   @Test
   public void testChecksum32() throws IOException {
     // test Adler32