Working till 01-12-2025 with around 80% accuracy

Author: khemchand-zetta

convert_hf_to_gguf.py

@@ -643,6 +643,7 @@ def print_registered_models(cls):
     @classmethod
     def from_model_architecture(cls, arch: str, model_type = ModelType.TEXT) -> type[ModelBase]:
         try:
+            print(cls._model_classes)
             return cls._model_classes[model_type][arch]
         except KeyError:
             raise NotImplementedError(f'Architecture {arch!r} not supported!') from None
@@ -4457,6 +4458,48 @@ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
         yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_LONG), torch.tensor(long_factors, dtype=torch.float32))
         yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT), torch.tensor(short_factors, dtype=torch.float32))
 
+@ModelBase.register("Phi3VForCausalLM")
+class Phi3VisionModel(Phi3MiniModel):
+    """
+    GGUF converter for Phi-3 Vision (Text Part Only).
+
+    This strips out the vision encoder weights and metadata, creating a
+    standard Phi-3 GGUF file that can be paired with an external mmproj file.
+    """
+
+    # CRITICAL: Use PHI3, not PHI3_VISION.
+    # This tells llama.cpp to treat this as a standard text model.
+    model_arch = gguf.MODEL_ARCH.PHI3
+
+    def set_vocab(self):
+        return super().set_vocab()
+
+    def set_gguf_parameters(self):
+        # Only write standard text model parameters (context length, embedding size, etc.)
+        super().set_gguf_parameters()
+
+    def generate_extra_tensors(self):
+        # This handles the 'su' RoPE scaling factors (long/short) defined in Phi3MiniModel
+        yield from super().generate_extra_tensors()
+
+    def modify_tensors(
+            self,
+            data_torch: Tensor,
+            name: str,
+            bid: int | None,
+    ) -> Iterable[tuple[str, Tensor]]:
+
+        # The prefix for all vision-related weights in Phi-3-Vision
+        VISION_PREFIX = "model.vision_embed_tokens."
+
+        # 1. If it is a vision tensor, SKIP IT completely.
+        # We do not want these weights in the text model file.
+        if name.startswith(VISION_PREFIX):
+            return
+
+        # 2. If it is a text tensor, delegate to the standard Phi-3 logic.
+        # This handles token_embd, layers, output, norms, etc.
+        yield from super().modify_tensors(data_torch, name, bid)
 
 @ModelBase.register("PhiMoEForCausalLM")
 class PhiMoeModel(Phi3MiniModel):
@@ -7936,7 +7979,7 @@ def set_gguf_parameters(self):
         self.gguf_writer.add_context_length(self.hparams.get("seq_length", n_embed))
         self.gguf_writer.add_embedding_length(n_embed)
         self.gguf_writer.add_feed_forward_length(self.hparams.get("ffn_hidden_size", self.hparams.get("intermediate_size", 4 * n_embed)))
-        self.gguf_writer.add_block_count(self.hparams.get("num_layers", self.hparams["num_hidden_layers"]))
+        self.gguf_writer.add_block_count(self.hparams.get("num_layers", self.hparams.get("num_hidden_layers", 0)))
         self.gguf_writer.add_head_count(n_head)
         self.gguf_writer.add_head_count_kv(n_head_kv)
         self.gguf_writer.add_layer_norm_rms_eps(self.hparams.get("layernorm_epsilon",1e-5))
@@ -10143,6 +10186,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
     # maybe we should fallback to text model's arch in that case, since not many models have both
     text_config = hparams.get("text_config", {})
     vision_config = hparams.get("vision_config", {})
+    print(hparams.get("architectures"))
     arch = None
     if (arches := hparams.get("architectures")) is not None and len(arches) > 0:
         arch = arches[0]

ggml/src/ggml-cpu/ggml-cpu.c

@@ -193,6 +193,11 @@ typedef pthread_t ggml_thread_t;
 #include <TargetConditionals.h>
 #endif
 
+#include <stdatomic.h>
+
+static _Atomic uint64_t ggml_op_us[GGML_OP_COUNT];
+static _Atomic uint64_t ggml_op_calls[GGML_OP_COUNT];
+
 static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {
     [GGML_TYPE_F32] = {
         .from_float               = (ggml_from_float_t) ggml_cpu_fp32_to_fp32,
@@ -2864,6 +2869,44 @@ struct ggml_cplan ggml_graph_plan(
     return cplan;
 }
 
+// static thread_ret_t ggml_graph_compute_thread(void * data) {
+//     struct ggml_compute_state * state = (struct ggml_compute_state *) data;
+//     struct ggml_threadpool    * tp    = state->threadpool;
+//
+//     const struct ggml_cgraph * cgraph = tp->cgraph;
+//     const struct ggml_cplan  * cplan  = tp->cplan;
+//
+//     set_numa_thread_affinity(state->ith);
+//
+//     struct ggml_compute_params params = {
+//         /*.ith       =*/ state->ith,
+//         /*.nth       =*/ atomic_load_explicit(&tp->n_threads_cur, memory_order_relaxed),
+//         /*.wsize     =*/ cplan->work_size,
+//         /*.wdata     =*/ cplan->work_data,
+//         /*.threadpool=*/ tp,
+//     };
+//
+//     for (int node_n = 0; node_n < cgraph->n_nodes && atomic_load_explicit(&tp->abort, memory_order_relaxed) != node_n; node_n++) {
+//         struct ggml_tensor * node = cgraph->nodes[node_n];
+//
+//         ggml_compute_forward(&params, node);
+//
+//         if (state->ith == 0 && cplan->abort_callback &&
+//                 cplan->abort_callback(cplan->abort_callback_data)) {
+//             atomic_store_explicit(&tp->abort, node_n + 1, memory_order_relaxed);
+//             tp->ec    = GGML_STATUS_ABORTED;
+//         }
+//
+//         if (node_n + 1 < cgraph->n_nodes) {
+//             ggml_barrier(state->threadpool);
+//         }
+//     }
+//
+//     ggml_barrier(state->threadpool);
+//
+//     return 0;
+// }
+
 static thread_ret_t ggml_graph_compute_thread(void * data) {
     struct ggml_compute_state * state = (struct ggml_compute_state *) data;
     struct ggml_threadpool    * tp    = state->threadpool;
@@ -2884,21 +2927,25 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
     for (int node_n = 0; node_n < cgraph->n_nodes && atomic_load_explicit(&tp->abort, memory_order_relaxed) != node_n; node_n++) {
         struct ggml_tensor * node = cgraph->nodes[node_n];
 
+        uint64_t t0 = ggml_time_us();
         ggml_compute_forward(&params, node);
+        uint64_t dt = ggml_time_us() - t0;
+
+        atomic_fetch_add_explicit(&ggml_op_us[node->op], dt, memory_order_relaxed);
+        atomic_fetch_add_explicit(&ggml_op_calls[node->op], 1, memory_order_relaxed);
 
         if (state->ith == 0 && cplan->abort_callback &&
                 cplan->abort_callback(cplan->abort_callback_data)) {
             atomic_store_explicit(&tp->abort, node_n + 1, memory_order_relaxed);
             tp->ec    = GGML_STATUS_ABORTED;
-        }
+                }
 
         if (node_n + 1 < cgraph->n_nodes) {
             ggml_barrier(state->threadpool);
         }
     }
 
     ggml_barrier(state->threadpool);
-
     return 0;
 }
 
@@ -3201,6 +3248,33 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cpl
         ggml_threadpool_free(threadpool);
     }
 
+    // printf("\n========= GGML OP PERF =========\n");
+    // for (int i = 0; i < GGML_OP_COUNT; i++) {
+    //     uint64_t us    = atomic_load(&ggml_op_us[i]);
+    //     uint64_t calls = atomic_load(&ggml_op_calls[i]);
+    //     if (calls == 0) continue;
+    //
+    //     printf("%-16s : %8llu us   %6llu calls   avg %6llu us\n",
+    //            ggml_op_name(i),
+    //            (unsigned long long)us,
+    //            (unsigned long long)calls,
+    //            (unsigned long long)(us / calls));
+    // }
+    // printf("================================\n\n");
+
+    // printf("\n");
+    // for (int i = 0; i < GGML_OP_COUNT; i++) {
+    //     uint64_t us    = atomic_load(&ggml_op_us[i]);
+    //     uint64_t calls = atomic_load(&ggml_op_calls[i]);
+    //     if (calls == 0) continue;
+    //
+    //     printf("%-16s,%8llu us,%6llu,%6llu us,",
+    //            ggml_op_name(i),
+    //            (unsigned long long)us,
+    //            (unsigned long long)calls,
+    //            (unsigned long long)(us / calls));
+    // }
+
     return ret;
 }
 

gguf-py/gguf/constants.py

@@ -356,6 +356,7 @@ class MODEL_ARCH(IntEnum):
     QWEN3VLMOE       = auto()
     PHI2             = auto()
     PHI3             = auto()
+    PHI3_VISION      = auto()
     PHIMOE           = auto()
     PLAMO            = auto()
     PLAMO2           = auto()
@@ -723,6 +724,7 @@ class MODEL_TENSOR(IntEnum):
     MODEL_ARCH.QWEN3VLMOE:       "qwen3vlmoe",
     MODEL_ARCH.PHI2:             "phi2",
     MODEL_ARCH.PHI3:             "phi3",
+    MODEL_ARCH.PHI3_VISION:      "phi3_vision",
     MODEL_ARCH.PHIMOE:           "phimoe",
     MODEL_ARCH.PLAMO:            "plamo",
     MODEL_ARCH.PLAMO2:           "plamo2",
@@ -1670,6 +1672,22 @@ class MODEL_TENSOR(IntEnum):
         MODEL_TENSOR.FFN_DOWN,
         MODEL_TENSOR.FFN_UP,
     ],
+    MODEL_ARCH.PHI3_VISION: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ROPE_FACTORS_LONG,
+        MODEL_TENSOR.ROPE_FACTORS_SHORT,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_QKV,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_UP,
+    ],
     MODEL_ARCH.PHIMOE: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT_NORM,

src/llama-arch.cpp

@@ -36,6 +36,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_QWEN3VLMOE,       "qwen3vlmoe"       },
     { LLM_ARCH_PHI2,             "phi2"             },
     { LLM_ARCH_PHI3,             "phi3"             },
+    { LLM_ARCH_PHI3_VISION,      "phi3_vision"      },
     { LLM_ARCH_PHIMOE,           "phimoe"           },
     { LLM_ARCH_PLAMO,            "plamo"            },
     { LLM_ARCH_PLAMO2,           "plamo2"           },

src/llama-arch.h

@@ -40,6 +40,7 @@ enum llm_arch {
     LLM_ARCH_QWEN3VLMOE,
     LLM_ARCH_PHI2,
     LLM_ARCH_PHI3,
+    LLM_ARCH_PHI3_VISION,
     LLM_ARCH_PHIMOE,
     LLM_ARCH_PLAMO,
     LLM_ARCH_PLAMO2,

src/llama-graph.h

@@ -473,7 +473,7 @@ class llm_graph_result {
 
     virtual ~llm_graph_result() = default;
 
-    ggml_tensor * get_tokens()      const { return t_tokens; }
+    auto          get_tokens() const -> ggml_tensor * { return t_tokens; }
     ggml_tensor * get_logits()      const { return t_logits; }
     ggml_tensor * get_embd()        const { return t_embd; }
     ggml_tensor * get_embd_pooled() const { return t_embd_pooled; }

tools/mtmd/clip-impl.h

@@ -39,6 +39,11 @@
 #define KEY_FEATURE_LAYER       "clip.vision.feature_layer"
 #define KEY_PROJ_SCALE_FACTOR   "clip.vision.projector.scale_factor"
 #define KEY_SPATIAL_MERGE_SIZE  "clip.vision.spatial_merge_size"
+// [NEW] Phi-3-Vision Specific Keys
+#define KEY_PHI3_HD_ORDER         "clip.vision.hd_transform_order"      // Stores "sub_glb"
+#define KEY_PHI3_NUM_IMG_TOKENS   "clip.vision.num_img_tokens"          // Stores 144
+#define KEY_PHI3_USE_HD           "clip.vision.use_hd_transform"        // Stores true
+#define KEY_PHI3_WITH_SEP         "clip.vision.with_learnable_separator" // Stores true
 #define KEY_IS_DEEPSTACK_LAYERS "clip.vision.is_deepstack_layers"
 
 #define KEY_MM_PATCH_MERGE_TYPE   "clip.vision.mm_patch_merge_type"
@@ -86,6 +91,21 @@
 #define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s"
 #define TN_MVLM_PROJ_PEG   "mm.model.peg.%d.%s"
 #define TN_IMAGE_NEWLINE   "model.image_newline"
+
+// [NEW] Phi-3-Vision Specific Tensors
+// Mapping for: model.vision_embed_tokens.glb_GN
+#define TN_PHI3_GLB_GN      "v.glb_GN"
+// Mapping for: model.vision_embed_tokens.sub_GN
+#define TN_PHI3_SUB_GN      "v.sub_GN"
+
+// [NEW] Projector Mapping
+// Your tensor map shows "model.vision_embed_tokens.img_projection.0.weight"
+// and "model.vision_embed_tokens.img_projection.2.weight".
+// This confirms it is a 2-layer MLP (Layer 0 = Linear, Layer 1 = GELU (implicit), Layer 2 = Linear).
+// We can reuse TN_LLAVA_PROJ ("mm.%d.%s") or define a specific one if the conversion script names them uniquely.
+// To be safe and specific:
+#define TN_PHI3_PROJ_MLP    "mm.phi3_mlp.%d.%s"
+
 #define TN_MM_INP_NORM     "mm.input_norm.weight"
 #define TN_MM_INP_NORM_B   "mm.input_norm.bias"
 #define TN_MM_INP_PROJ     "mm.input_projection.weight" // gemma3
@@ -156,6 +176,7 @@ enum projector_type {
     PROJECTOR_TYPE_LIGHTONOCR,
     PROJECTOR_TYPE_COGVLM,
     PROJECTOR_TYPE_JANUS_PRO,
+    PROJECTOR_TYPE_PHI3_V,
     PROJECTOR_TYPE_UNKNOWN,
 };
 
@@ -182,6 +203,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
     { PROJECTOR_TYPE_LIGHTONOCR,"lightonocr"},
     { PROJECTOR_TYPE_COGVLM,    "cogvlm"},
     { PROJECTOR_TYPE_JANUS_PRO, "janus_pro"},
+    { PROJECTOR_TYPE_PHI3_V,    "phi3_v"},
 };
 
 static projector_type clip_projector_type_from_string(const std::string & str) {