Remove deprecated methods for Q8_0 tensor loading and conversion to FP32

Author: orionpapadakis

src/main/java/org/beehive/gpullama3/model/loader/ModelLoader.java

@@ -145,31 +145,6 @@ public static TornadoTensor[] loadArrayOfTornadoTensors(int size, IntFunction<GG
         return array;
     }
 
-    /**
-     * Load a tensor and manually convert to FP32 (FloatArray).
-     * Used for embeddings that currently are treated as FP32.
-     * TODO: it is ultra-slow and should be removed
-     */
-    public static TornadoTensor loadTornadoTensorAsFP32(GGMLTensorEntry entry) {
-        TornadoTensor tensor = loadTornadoTensor(entry);
-        return switch (tensor.type()) {
-            case F32 -> tensor;
-            case F16 -> {
-                HalfFloatArray tensorHFA = tensor.asHalfFloatArray();
-                int numOfElements = tensorHFA.getSize();
-                FloatArray tensorFA = new FloatArray(numOfElements);
-                for (int i = 0; i < numOfElements; i++) {
-                    tensorFA.set(i, tensorHFA.get(i).getFloat32());
-                }
-                yield new FP32TornadoTensor(tensorFA);
-            }
-            case Q8_0 -> Q8_0TornadoTensor.createAsFP32(entry);
-            default -> {
-                throw new UnsupportedOperationException("Unsupported tensor type: " + tensor.type());
-            }
-        };
-    }
-
     // Helper methods
 
     public static FloatArray[] loadArrayAsFloatArray(int size, IntFunction<GGMLTensorEntry> getTensorEntry) {
@@ -188,14 +163,6 @@ public static HalfFloatArray[] loadArrayAsHalfFloatArray(int size, IntFunction<G
         return array;
     }
 
-    public static Q8_0TornadoTensor[] loadArrayAsQ8_0TornadoTensor(int size, IntFunction<GGMLTensorEntry> getTensorEntry) {
-        Q8_0TornadoTensor[] array = new Q8_0TornadoTensor[size];
-        for (int i = 0; i < size; i++) {
-            array[i] = Q8_0TornadoTensor.createAsQ8_0(getTensorEntry.apply(i));
-        }
-        return array;
-    }
-
     public static FloatArray floatBufferToFloatArray(GGMLTensorEntry tensorEntry) {
         if (tensorEntry.ggmlType() == GGMLType.F32) {
             FloatBuffer buffer = tensorEntry.memorySegment().asByteBuffer().order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer();

src/main/java/org/beehive/gpullama3/tensor/tornado/Q8_0TornadoTensor.java

@@ -94,125 +94,4 @@ public float getFloat(int index) {
         return quant * scale;
     }
 
-    /**
-     * Creates a Q8_0TornadoTensor from a GGMLTensorEntry (original implementation).
-     */
-    public static Q8_0TornadoTensor createAsQ8_0(GGMLTensorEntry entry) {
-        if (entry.ggmlType() != GGMLType.Q8_0) {
-            throw new IllegalArgumentException("Expected Q8_0 tensor, got: " + entry.ggmlType() + " for tensor: " + entry.name());
-        }
-
-        int[] shape = entry.shape();
-        int size = FloatTensor.numberOfElements(shape);
-        int numBlocks = size / GGMLType.Q8_0.getBlockSize();
-
-        if (size % GGMLType.Q8_0.getBlockSize() != 0) {
-            throw new IllegalArgumentException("Q8_0 tensor size must be multiple of " + GGMLType.Q8_0.getBlockSize() + ", got: " + size + " for tensor: " + entry.name());
-        }
-
-        // TODO: fix Q8_0 loading in tornado layoyt
-        //  currently we end up to hack it by removing
-        //  tornado header from memory segment
-        MemorySegment q8Segment = entry.memorySegment().asSlice(TornadoNativeArray.ARRAY_HEADER);
-
-        // allocate the arrays for quantized data (int8) and scales (fp16)
-        HalfFloatArray scales = new HalfFloatArray(numBlocks);
-        Int8Array quants = new Int8Array(size);
-
-        // unpack Q8_0 blocks: [2 bytes fp16 scale][32 bytes int8 quants]
-        ValueLayout.OfShort shortLayout = ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
-        ValueLayout.OfByte byteLayout = ValueLayout.JAVA_BYTE;
-
-        // element-wise copy and unpack from MemorySegment to HalfFloatArray scales and Int8Array quants
-        // use parallel streams and unroll inner loop for better performance
-        IntStream.range(0, numBlocks)
-                .parallel()
-                .forEach(block -> {
-                    // TODO: use GGML type method for the 34L size
-                    long blockOffset = block * 34L;  // 34 bytes per block
-
-                    // read fp16 scale (first 2 bytes of block)
-                    short scaleRaw = q8Segment.get(shortLayout, blockOffset);
-                    scales.set(block, new HalfFloat(scaleRaw));
-                    int blockStart = block * 32;
-
-                    // read 32 int8 quantized values (remaining bytes of block)
-                    // TODO: use GGML type method for the 32 size
-                    for (int i = 0; i < 32; i += 4) {
-                        // unroll inner loop for better performance
-                        byte q0 = q8Segment.get(byteLayout, blockOffset + 2 + i);
-                        byte q1 = q8Segment.get(byteLayout, blockOffset + 2 + i + 1);
-                        byte q2 = q8Segment.get(byteLayout, blockOffset + 2 + i + 2);
-                        byte q3 = q8Segment.get(byteLayout, blockOffset + 2 + i + 3);
-
-                        quants.set(blockStart + i,     q0);
-                        quants.set(blockStart + i + 1, q1);
-                        quants.set(blockStart + i + 2, q2);
-                        quants.set(blockStart + i + 3, q3);
-                    }
-                });
-
-        return new Q8_0TornadoTensor(size, scales, quants, q8Segment);
-    }
-
-    /**
-     * Creates a Q8_0TornadoTensor formulated as FP32TornadoTensor object from a GGMLTensorEntry.
-     * NOTE: Hack implementation to comply with FP32 inference.
-     */
-    public static FP32TornadoTensor createAsFP32(GGMLTensorEntry entry) {
-        if (entry.ggmlType() != GGMLType.Q8_0) {
-            throw new IllegalArgumentException("Expected Q8_0 tensor, got: " + entry.ggmlType() + " for tensor: " + entry.name());
-        }
-
-        int[] shape = entry.shape();
-        int size = FloatTensor.numberOfElements(shape);
-        int numBlocks = size / GGMLType.Q8_0.getBlockSize();
-
-        if (size % GGMLType.Q8_0.getBlockSize() != 0) {
-            throw new IllegalArgumentException("Q8_0 tensor size must be multiple of " + GGMLType.Q8_0.getBlockSize() + ", got: " + size + " for tensor: " + entry.name());
-        }
-
-        // TODO: fix Q8_0 loading in tornado layoyt
-        //  currently we end up to hack it by removing
-        //  tornado header from memory segment
-        MemorySegment q8Segment = entry.memorySegment().asSlice(TornadoNativeArray.ARRAY_HEADER);
-
-        // allocate the FloatArray to store the result
-        FloatArray floatArray = new FloatArray(size);
-
-        // unpack Q8_0 blocks: [2 bytes fp16 scale][32 bytes int8 quants]
-        ValueLayout.OfShort shortLayout = ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
-        ValueLayout.OfByte byteLayout = ValueLayout.JAVA_BYTE;
-
-        // element-wise dequantization and copy from MemorySegment to FloatArray
-        // use parallel streams and unroll inner loop for better performance
-        IntStream.range(0, numBlocks)
-                .parallel()
-                .forEach(block -> {
-                    // TODO: use GGML type method for the 34L size
-                    long blockOffset = block * 34L;  // 34 bytes per block
-
-                    // read fp16 scale (first 2 bytes of block) and convert to float
-                    short scaleRaw = q8Segment.get(shortLayout, blockOffset);
-                    float scale = Float.float16ToFloat(scaleRaw);
-                    int blockStart = block * 32;
-
-                    // read 32 int8 quantized values (remaining bytes of block)
-                    // TODO: use GGML type method for the 32 size
-                    for (int i = 0; i < 32; i += 4) {
-                        // unroll inner loop for better performance
-                        byte q0 = q8Segment.get(byteLayout, blockOffset + 2 + i);
-                        byte q1 = q8Segment.get(byteLayout, blockOffset + 2 + i + 1);
-                        byte q2 = q8Segment.get(byteLayout, blockOffset + 2 + i + 2);
-                        byte q3 = q8Segment.get(byteLayout, blockOffset + 2 + i + 3);
-
-                        floatArray.set(blockStart + i,     q0 * scale);
-                        floatArray.set(blockStart + i + 1, q1 * scale);
-                        floatArray.set(blockStart + i + 2, q2 * scale);
-                        floatArray.set(blockStart + i + 3, q3 * scale);
-                    }
-                });
-
-        return new FP32TornadoTensor(floatArray);
-    }
 }