FP8 K is inferring again.

Author: createthis

ggml/src/ggml-cpu/ggml-cpu.c

@@ -2359,6 +2359,11 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
             {
                 n_tasks = n_threads;
             } break;
+        case GGML_OP_KV_DSMLA_PACK:
+            {
+                // trivial metadata op for FP8 KV; handled only on CUDA backend
+                n_tasks = 1;
+            } break;
         case GGML_OP_NONE:
             {
                 n_tasks = 1;

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -3803,6 +3803,19 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
                 if (op->ne[2] != 1 || op->ne[3] != 1) return false;
                 return ggml_is_contiguous(a);
             } break;
+        case GGML_OP_KV_DSMLA_PACK:
+            {
+                const struct ggml_tensor * k_lr  = op->src[0];
+                const struct ggml_tensor * k_idx = op->src[1];
+                const struct ggml_tensor * blob  = op->src[2];
+                if (!k_lr || !k_idx || !blob) return false;
+                if (k_lr->type  != GGML_TYPE_F32) return false;
+                if (k_idx->type != GGML_TYPE_I64) return false;
+                if (blob->type  != GGML_TYPE_I8)  return false;
+                if (!ggml_is_contiguous(k_lr)) return false;
+                if (k_lr->ne[1] != 1) return false;
+                return true;
+            } break;
         case GGML_OP_GLU:
             switch (ggml_get_glu_op(op)) {
                 case GGML_GLU_OP_REGLU:

src/llama-kv-cache-fp8.cpp

@@ -1,4 +1,5 @@
 #include "llama-kv-cache-fp8.h"
+#include "ggml-backend.h"
 
 #include "llama-impl.h"
 #include "llama-io.h"
@@ -813,6 +814,11 @@ ggml_tensor * llama_kv_cache_fp8::get_k(ggml_context * ctx, int32_t il, uint32_t
     if (lyr == nullptr || lyr->k_blob == nullptr) {
         return nullptr;
     }
+    // If K blob is device-resident, we cannot safely dereference it on host during graph build.
+    // In that case, skip FP8-derived K and let callers fall back to the float KV cache.
+    if (lyr->k_blob->buffer && !ggml_backend_buffer_is_host(lyr->k_blob->buffer)) {
+        return nullptr;
+    }
 
     const uint32_t ns = sinfo.s1 - sinfo.s0 + 1;
     const uint32_t kv_size = get_size();
@@ -881,6 +887,11 @@ ggml_tensor * llama_kv_cache_fp8::cpy_k(ggml_context * ctx, ggml_tensor * k_cur,
     if (lyr == nullptr || lyr->k_blob == nullptr) {
         return nullptr;
     }
+    // If K blob is device-resident, we cannot safely dereference it on host during graph build.
+    // In that case, skip FP8-derived K and let callers fall back to the float KV cache.
+    if (lyr->k_blob->buffer && !ggml_backend_buffer_is_host(lyr->k_blob->buffer)) {
+        return nullptr;
+    }
 
     const uint32_t ns = sinfo.s1 - sinfo.s0 + 1;
     const uint32_t kv_size = get_size();