refactor v2 code

karlnicholas · karlnicholas · commit 3469ecf94b3d · 2025-10-20T14:59:07.000-04:00
diff --git a/src/main/java/org/hdf5javalib/hdffile/dataobjects/messages/FilterPipelineMessage.java b/src/main/java/org/hdf5javalib/hdffile/dataobjects/messages/FilterPipelineMessage.java
@@ -22,23 +22,23 @@
  *
  * <h2>Structure</h2>
  * <ul>
- *   <li><b>Version (1 byte)</b>: The version of the fill value format.</li>
- *   <li><b>Space Allocation Time (1 byte, version-dependent)</b>: When space for
- *       fill values is allocated (Early, Late, or Incremental).</li>
- *   <li><b>Fill Value Write Time (1 byte, version-dependent)</b>: When the fill
- *       value is written (on creation or first write).</li>
- *   <li><b>Fill Value Defined Flag (1 byte)</b>: Indicates if a user-defined fill
- *       value is provided.</li>
- *   <li><b>Fill Value Size (4 bytes, optional)</b>: The size of the fill value in bytes.</li>
- *   <li><b>Fill Value Data (variable, optional)</b>: The actual fill value, matching
- *       the dataset's datatype.</li>
+ * <li><b>Version (1 byte)</b>: The version of the fill value format.</li>
+ * <li><b>Space Allocation Time (1 byte, version-dependent)</b>: When space for
+ * fill values is allocated (Early, Late, or Incremental).</li>
+ * <li><b>Fill Value Write Time (1 byte, version-dependent)</b>: When the fill
+ * value is written (on creation or first write).</li>
+ * <li><b>Fill Value Defined Flag (1 byte)</b>: Indicates if a user-defined fill
+ * value is provided.</li>
+ * <li><b>Fill Value Size (4 bytes, optional)</b>: The size of the fill value in bytes.</li>
+ * <li><b>Fill Value Data (variable, optional)</b>: The actual fill value, matching
+ * the dataset's datatype.</li>
  * </ul>
  *
  * <h2>Usage</h2>
  * <ul>
- *   <li>Defining default values for missing or newly allocated data.</li>
- *   <li>Ensuring consistency when reading uninitialized portions of a dataset.</li>
- *   <li>Improving dataset integrity in applications requiring structured default data.</li>
+ * <li>Defining default values for missing or newly allocated data.</li>
+ * <li>Ensuring consistency when reading uninitialized portions of a dataset.</li>
+ * <li>Improving dataset integrity in applications requiring structured default data.</li>
  * </ul>
  *
  * @see HdfMessage
@@ -75,7 +75,7 @@ public FilterPipelineMessage(
             int flags,
             int sizeMessageData
     ) {
-        super(MessageType.FILL_VALUE_MESSAGE, sizeMessageData, flags);
+        super(MessageType.FILTER_PIPELINE_MESSAGE, sizeMessageData, flags);
         this.version = version;
         this.numberOfFilters = numberOfFilters;
         this.filterDescriptions = filterDescriptions;
@@ -104,22 +104,23 @@ public ByteBufferDeflater getDeflater() {
     public static HdfMessage parseHeaderMessage(int flags, byte[] data, HdfDataFile hdfDataFile) {
         ByteBuffer buffer = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN);
 
-        // Parse the first 4 bytes
         int version = Byte.toUnsignedInt(buffer.get());
         int numberOfFilters = Byte.toUnsignedInt(buffer.get());
-        buffer.position(buffer.position() + 2);
-        buffer.position(buffer.position() + 4);
 
-        // Handle Version 2+ behavior and fillValueDefined flag
         if (version == 1) {
-            List<FilterDescription> filterDescriptions = new ArrayList<>();
-            for(int i = 0; i < numberOfFilters; i++) {
-                filterDescriptions.add(FilterDescription.parseFilterDescription(buffer));
-            }
-            // Return a constructed instance of FillValueMessage
-            return new FilterPipelineMessage(version, numberOfFilters, filterDescriptions, flags, data.length);
+            // Skip 6 bytes of reserved space for V1
+            buffer.position(buffer.position() + 6);
+        } else if (version != 2) {
+            // Version 2 has no extra header bytes to skip.
+            throw new UnsupportedOperationException("Filter pipeline message version not supported: " + version);
+        }
+
+        List<FilterDescription> filterDescriptions = new ArrayList<>();
+        for (int i = 0; i < numberOfFilters; i++) {
+            filterDescriptions.add(FilterDescription.parseFilterDescription(buffer, version));
         }
-        throw new UnsupportedOperationException("Version not supported: "  + version);
+
+        return new FilterPipelineMessage(version, numberOfFilters, filterDescriptions, flags, data.length);
     }
 
     /**
@@ -173,21 +174,30 @@ public String toString() {
                     '}';
         }
 
-        public static FilterDescription parseFilterDescription(ByteBuffer buffer) {
+        public static FilterDescription parseFilterDescription(ByteBuffer buffer, int version) {
             int filterIndentification = Short.toUnsignedInt(buffer.getShort());
-            int nameLength = Short.toUnsignedInt(buffer.getShort());
+            int nameLength = 0;
+            if ( version == 1 || (version == 2 && filterIndentification > 256)) {
+                nameLength = Short.toUnsignedInt(buffer.getShort());
+            }
             byte[] flagBytes = new byte[2];
             buffer.get(flagBytes);
             BitSet flags = BitSet.valueOf(flagBytes);
+
             int numberOfValuesForClientData = Short.toUnsignedInt(buffer.getShort());
-            byte[] nameBytes = new byte[nameLength];
-            buffer.get(nameBytes);
-            String name = new String(HdfReadUtils.trimZeroBytes(nameBytes));
+
+            String name = null;
+            if( nameLength > 0) {
+                byte[] nameBytes = new byte[nameLength];
+                buffer.get(nameBytes);
+                name = new String(HdfReadUtils.trimZeroBytes(nameBytes));
+            }
+
             int[] clientData = new int[numberOfValuesForClientData];
             for(int i = 0; i < numberOfValuesForClientData; i++) {
                 clientData[i] = buffer.getInt();
             }
-            if ( numberOfValuesForClientData%2 != 0) {
+            if ( version == 1 && numberOfValuesForClientData%2 != 0) {
                 int[] padding = new int[1];
                 padding[0] = buffer.getInt();
             }
diff --git a/src/main/java/org/hdf5javalib/hdffile/infrastructure/fractalheap/FractalHeap.java b/src/main/java/org/hdf5javalib/hdffile/infrastructure/fractalheap/FractalHeap.java
@@ -43,12 +43,11 @@ public static FractalHeap read(SeekableByteChannel channel, long position, HdfDa
             filteredSize = header.filteredRootDirectSize;
             filterMask = header.filterMaskRoot;
         }
-        long expectedBlockOffset = 0;
         Block root;
         if (nrows == 0) {
-            root = readDirectBlock(channel, header, rootAddress, sizeOfOffset, expectedBlockOffset, filteredSize, filterMask);
+            root = readDirectBlock(channel, header, rootAddress, sizeOfOffset, filteredSize, filterMask);
         } else {
-            root = readIndirectBlock(channel, header, sizeOfOffset, sizeOfLength, rootAddress, expectedBlockOffset, -1, nrows);
+            root = readIndirectBlock(channel, header, sizeOfOffset, sizeOfLength, rootAddress, -1, nrows);
         }
         FractalHeap heap = new FractalHeap();
         heap.header = header;
@@ -273,7 +272,7 @@ private static Filter parseV2Filter(ByteBuffer bb) {
         return f;
     }
 
-    private static Block readDirectBlock(SeekableByteChannel channel, FractalHeapHeader header, HdfFixedPoint address, FixedPointDatatype sizeOfOffset, long expectedBlockOffset, HdfFixedPoint filteredSize, long filterMask) throws IOException, InvocationTargetException, InstantiationException, IllegalAccessException {
+    private static Block readDirectBlock(SeekableByteChannel channel, FractalHeapHeader header, HdfFixedPoint address, FixedPointDatatype sizeOfOffset, HdfFixedPoint filteredSize, long filterMask) throws IOException, InvocationTargetException, InstantiationException, IllegalAccessException {
         if (address.isUndefined()) {
             throw new IOException("Invalid direct block address");
         }
@@ -299,15 +298,12 @@ private static Block readDirectBlock(SeekableByteChannel channel, FractalHeapHea
             throw new IOException("Invalid heap header address in direct block");
         }
         HdfFixedPoint blockOffset = HdfReadUtils.readHdfFixedPointFromBuffer(header.offsetBytes, headerBuffer);
-        if (blockOffset.getInstance(Long.class) != expectedBlockOffset) {
-            throw new IOException("Block offset mismatch");
-        }
         headerBuffer.position(4 + 1 + sizeOfOffset.getSize()+header.offsetBytes.getSize());
         long checksum = 0;
         if (header.checksumDirect) {
             checksum = Integer.toUnsignedLong(headerBuffer.getInt()); // checksum, not verifying
         }
-        long blockSize = getBlockSize(header, expectedBlockOffset);
+        long blockSize = getBlockSize(header, blockOffset.getInstance(Long.class));
         long dataSize;
         if (!filteredSize.isUndefined()) {
             dataSize = filteredSize.getInstance(Long.class);
@@ -317,16 +313,23 @@ private static Block readDirectBlock(SeekableByteChannel channel, FractalHeapHea
         if (dataSize < 0) {
             throw new IOException("Invalid data size in direct block");
         }
-        headerBuffer = ByteBuffer.allocate((int) dataSize).order(ByteOrder.LITTLE_ENDIAN);
+        // Determine the maximum number of bytes that can be read
+        long fSize = channel.size();
+        long fPosition = channel.position();
+        long bytesRemainingInFile = fSize - fPosition;
+
+        // The actual size to read is the smaller of the two values
+        long actualReadSize = Math.min(dataSize, bytesRemainingInFile);
+        headerBuffer = ByteBuffer.allocate((int) actualReadSize).order(ByteOrder.LITTLE_ENDIAN);
         bytesRead = channel.read(headerBuffer);
-        if ( bytesRead != dataSize)
+        if ( bytesRead != actualReadSize)
             throw new IllegalStateException();
         headerBuffer.flip();
 
         byte[] data = headerBuffer.array();
         DirectBlock db = new DirectBlock();
         db.blockOffset = blockOffset.getInstance(Long.class);
-        db.blockSize = blockSize;
+        db.blockSize = actualReadSize;
         db.data = data;
         db.filterMask = filterMask;
         db.checksum = checksum;
@@ -367,7 +370,7 @@ private static int calculateChildrenEntriesSize(FractalHeapHeader header, int nr
         return entriesSize;
     }
 
-    private static void readAndVerifyIndirectBlockHeader(ByteBuffer blockBuffer, long expectedBlockOffset, FixedPointDatatype sizeOfOffset, FixedPointDatatype offsetBytes) throws IOException, InvocationTargetException, InstantiationException, IllegalAccessException {
+    private static long readAndVerifyIndirectBlockHeader(ByteBuffer blockBuffer, FixedPointDatatype sizeOfOffset, FixedPointDatatype offsetBytes) throws IOException, InvocationTargetException, InstantiationException, IllegalAccessException {
         byte[] signatureBytes = new byte[4];
         blockBuffer.get(signatureBytes);
         String sig = new String(signatureBytes, StandardCharsets.US_ASCII);
@@ -386,14 +389,13 @@ private static void readAndVerifyIndirectBlockHeader(ByteBuffer blockBuffer, lon
         }
 
         HdfFixedPoint blockOffset = HdfReadUtils.readHdfFixedPointFromBuffer(offsetBytes, blockBuffer);
-        if (blockOffset.getInstance(Long.class) != expectedBlockOffset) {
-            throw new IOException("Block offset mismatch");
-        }
+
+        return blockOffset.getInstance(Long.class);
     }
 
     private static List<ChildInfo> parseChildInfos(ByteBuffer blockBuffer, FractalHeapHeader header, int nrows, long startOffset, FixedPointDatatype sizeOfOffset, FixedPointDatatype sizeOfLength) throws InvocationTargetException, InstantiationException, IllegalAccessException, IOException {
         List<ChildInfo> childInfos = new ArrayList<>();
-        long currentOffset = startOffset;
+//        long currentOffset = startOffset;
         long startingBlockSize = header.startingBlockSize.getInstance(Long.class);
 
         for (short r = 0; r < nrows; r++) {
@@ -410,9 +412,9 @@ private static List<ChildInfo> parseChildInfos(ByteBuffer blockBuffer, FractalHe
                 }
 
                 if (!childAddress.isUndefined()) {
-                    childInfos.add(new ChildInfo(childAddress, currentOffset, isDirect, childFilteredSize, childFilterMask, rowBlockSize));
+                    childInfos.add(new ChildInfo(childAddress, isDirect, childFilteredSize, childFilterMask, rowBlockSize));
                 }
-                currentOffset += rowBlockSize;
+//                currentOffset += rowBlockSize;
             }
         }
         return childInfos;
@@ -423,17 +425,17 @@ private static List<Block> readChildren(SeekableByteChannel channel, FractalHeap
         for (ChildInfo info : childInfos) {
             Block child;
             if (info.isDirect) {
-                child = readDirectBlock(channel, header, info.address, sizeOfOffset, info.blockOffset, info.filteredSize, info.filterMask);
+                child = readDirectBlock(channel, header, info.address, sizeOfOffset, info.filteredSize, info.filterMask);
             } else {
-                child = readIndirectBlock(channel, header, sizeOfOffset, sizeOfLength, info.address, info.blockOffset, info.blockSize, 0);
+                child = readIndirectBlock(channel, header, sizeOfOffset, sizeOfLength, info.address, info.blockSize, 0);
             }
             children.add(child);
         }
         return children;
     }
     //</editor-fold>
 
-    private static Block readIndirectBlock(SeekableByteChannel channel, FractalHeapHeader header, FixedPointDatatype sizeOfOffset, FixedPointDatatype sizeOfLength, HdfFixedPoint address, long expectedBlockOffset, long iblockSize, int passedNrows) throws IOException, InvocationTargetException, InstantiationException, IllegalAccessException {
+    private static Block readIndirectBlock(SeekableByteChannel channel, FractalHeapHeader header, FixedPointDatatype sizeOfOffset, FixedPointDatatype sizeOfLength, HdfFixedPoint address, long iblockSize, int passedNrows) throws IOException, InvocationTargetException, InstantiationException, IllegalAccessException {
         if (address.isUndefined()) {
             throw new IOException("Invalid indirect block address");
         }
@@ -456,15 +458,15 @@ private static Block readIndirectBlock(SeekableByteChannel channel, FractalHeapH
         blockBuffer.flip();
 
         // 4. Verify the block's header fields. The buffer position is advanced past the header.
-        readAndVerifyIndirectBlockHeader(blockBuffer, expectedBlockOffset, sizeOfOffset, header.offsetBytes);
+        long fileBlockOffset = readAndVerifyIndirectBlockHeader(blockBuffer, sizeOfOffset, header.offsetBytes);
 
         // 5. Parse child information from the buffer. The buffer position is advanced past the entries.
-        List<ChildInfo> childInfos = parseChildInfos(blockBuffer, header, nrows, expectedBlockOffset, sizeOfOffset, sizeOfLength);
+        List<ChildInfo> childInfos = parseChildInfos(blockBuffer, header, nrows, fileBlockOffset, sizeOfOffset, sizeOfLength);
 
         // 6. Create the block object and read the checksum from the end of the buffer.
         IndirectBlock ib = new IndirectBlock();
         ib.nrows = nrows;
-        ib.blockOffset = expectedBlockOffset;
+        ib.blockOffset = fileBlockOffset;
         ib.checksum = Integer.toUnsignedLong(blockBuffer.getInt());
 
         // 7. Recursively read all children based on the parsed info.
@@ -575,15 +577,15 @@ public static class IndirectBlock extends Block {
 
     private static class ChildInfo {
         HdfFixedPoint address;
-        long blockOffset;
+//        long blockOffset;
         boolean isDirect;
         HdfFixedPoint filteredSize;
         long filterMask;
         long blockSize;
 
-        public ChildInfo(HdfFixedPoint address, long blockOffset, boolean isDirect, HdfFixedPoint filteredSize, long filterMask, long blockSize) {
+        public ChildInfo(HdfFixedPoint address, boolean isDirect, HdfFixedPoint filteredSize, long filterMask, long blockSize) {
             this.address = address;
-            this.blockOffset = blockOffset;
+//            this.blockOffset = blockOffset;
             this.isDirect = isDirect;
             this.filteredSize = filteredSize;
             this.filterMask = filterMask;
