[hack] Fix backwards compatibility with Q8_0

Author: orionpapadakis

src/main/java/org/beehive/gpullama3/model/loader/ModelLoader.java

@@ -153,7 +153,7 @@ public static TornadoTensor[] loadArrayOfTornadoTensors(int size, IntFunction<GG
     /**
      * Load a tensor and manually convert to FP32 (FloatArray).
      * Used for embeddings that currently are treated as FP32.
-     * TODO: it is ultra-slow and will be removed
+     * TODO: it is ultra-slow and should be removed
      */
     public static TornadoTensor loadTornadoTensorAsFP32(GGMLTensorEntry entry) {
         TornadoTensor tensor = loadTornadoTensor(entry);
@@ -168,6 +168,16 @@ public static TornadoTensor loadTornadoTensorAsFP32(GGMLTensorEntry entry) {
                 }
                 yield new FP32TornadoTensor(tensorFA);
             }
+            case Q8_0 -> {
+                Q8_0TornadoTensor tensorQ8_0 = Q8_0TornadoTensor.create(entry);
+                int numOfElements = tensorQ8_0.getSize();
+                FloatArray tensorFA = new FloatArray(numOfElements);
+                for(int i = 0; i < numOfElements; i++) {
+                    tensorFA.set(i, tensorQ8_0.getFloat(i));
+                }
+                yield new FP32TornadoTensor(tensorFA);
+
+            }
             default -> { throw new UnsupportedOperationException("Unsupported tensor type: " + tensor.type()); }
         };
     }

src/main/java/org/beehive/gpullama3/tensor/tornado/Q8_0TornadoTensor.java

@@ -6,23 +6,30 @@
 import uk.ac.manchester.tornado.api.types.HalfFloat;
 import uk.ac.manchester.tornado.api.types.arrays.HalfFloatArray;
 import uk.ac.manchester.tornado.api.types.arrays.Int8Array;
+import uk.ac.manchester.tornado.api.types.arrays.TornadoNativeArray;
 
 import java.lang.foreign.MemorySegment;
 import java.lang.foreign.ValueLayout;
 import java.nio.ByteOrder;
 
 public class Q8_0TornadoTensor extends TornadoTensor {
 
+    private final int size;
     private final HalfFloatArray scales;  // One per 32-element block
     private final Int8Array quants;       // Quantized int8 values
     private MemorySegment segment;
 
-    public Q8_0TornadoTensor(HalfFloatArray scales, Int8Array quants, MemorySegment segment) {
+    public Q8_0TornadoTensor(int size, HalfFloatArray scales, Int8Array quants, MemorySegment segment) {
+        this.size = size;
         this.scales = scales;
         this.quants = quants;
         this.segment = segment;
     }
 
+    public int getSize() {
+        return size;
+    }
+
     /**
      * Returns the scale factors for GPU kernels.
      *
@@ -77,7 +84,10 @@ public static Q8_0TornadoTensor create(GGMLTensorEntry entry) {
             throw new IllegalArgumentException("Q8_0 tensor size must be multiple of " + GGMLType.Q8_0.getBlockSize() + ", got: " + size + " for tensor: " + entry.name());
         }
 
-        MemorySegment q8Segment = entry.memorySegment();
+        // TODO: fix Q8_0 loading in tornado layoyt
+        //  currently we end up to hack it by removing
+        //  tornado header from memory segment
+        MemorySegment q8Segment = entry.memorySegment().asSlice(TornadoNativeArray.ARRAY_HEADER);
 
         // allocate the arrays for quantized data (int8) and scales (fp16)
         HalfFloatArray scales = new HalfFloatArray(numBlocks);
@@ -103,6 +113,6 @@ public static Q8_0TornadoTensor create(GGMLTensorEntry entry) {
             }
         }
 
-        return new Q8_0TornadoTensor(scales, quants, q8Segment);
+        return new Q8_0TornadoTensor(size, scales, quants, q8Segment);
     }
 }