Four fixes Qwen3.8-Flash-Next needs to work

common/arg.cpp

@@ -261,6 +261,13 @@ static void parse_tensor_buffer_overrides(const std::string & value, std::vector
         if (buft) {
             buft_list[ggml_backend_buft_name(buft)] = buft;
         }
+        // Also offer the device's pinned host buffer, so a tensor can deliberately be left in host
+        // memory and read in place. Worth it only for a large tensor that is gathered from rather
+        // than streamed; whether the backend will accept one as a kernel input is its own decision.
+        auto * host_buft = ggml_backend_dev_host_buffer_type(dev);
+        if (host_buft) {
+            buft_list[ggml_backend_buft_name(host_buft)] = host_buft;
+        }
     }
 
     for (const auto & override : string_split<std::string>(value, ',')) {

examples/speculative-simple/speculative-simple.cpp

@@ -75,6 +75,24 @@ int main(int argc, char ** argv) {
         }
 
         auto cparams = common_context_params_to_llama(params_dft);
+
+        // An MTP head only has the MTP graph; building it as an ordinary decoder walks the trunk
+        // tensors it does not carry. The server does the same at server-context.cpp. Both MTP
+        // types need this: the adaptive one only varies draft depth at runtime, so matching just
+        // COMMON_SPECULATIVE_TYPE_DRAFT_MTP left draft-mtp-adaptive loading the head as an
+        // ordinary decoder, failing, and crashing later on the null context.
+        const auto & spec_types = params.speculative.types;
+        const bool spec_mtp =
+            std::find(spec_types.begin(), spec_types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP)          != spec_types.end() ||
+            std::find(spec_types.begin(), spec_types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE) != spec_types.end();
+        if (spec_mtp) {
+            cparams.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
+            cparams.n_rs_seq = 0;
+            // an MTP head has no trunk and borrows the embeddings and lm head from the model it
+            // drafts for, so its context has to know which context that is
+            cparams.ctx_other = ctx_tgt;
+        }
+
         ctx_dft.reset(llama_init_from_model(model_dft.get(), cparams));
 
         params.speculative.draft.ctx_tgt = ctx_tgt;
@@ -224,13 +242,21 @@ int main(int argc, char ** argv) {
 
             //LOG_DBG("target batch: %s\n", string_from(ctx_tgt, batch_tgt).c_str());
 
-            llama_decode(ctx_tgt, batch_tgt);
+            const int32_t rc = llama_decode(ctx_tgt, batch_tgt);
+            if (rc != 0) {
+                LOG_ERR("%s: target decode failed, ret = %d\n", __func__, rc);
+                return 1;
+            }
         }
 
-        // evaluate the same batch with the draft model
-        {
-            // TODO: extend to support MTP, Eagle, etc. See server code for reference
-            llama_decode(ctx_dft.get(), batch_tgt);
+        // Hand the target's batch to the speculative layer and let it advance the draft in
+        // whatever way its kind requires. A standalone draft model replays the tokens, which is
+        // what this example used to do by hand; an MTP head instead needs the target's hidden
+        // states, and this call is the only thing that captures them. Doing it by hand left the
+        // head drafting from an empty state, which reads as fluent text that ignores the target.
+        if (!common_speculative_process(spec, batch_tgt)) {
+            LOG_ERR("%s: failed to advance the draft with the target's batch\n", __func__);
+            return 1;
         }
 
         // only save the sampler sampler state if we use checkpoints

src/llama-quant.cpp

@@ -332,6 +332,17 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
     quantize &= name.find("ssm_conv1d") == std::string::npos;
     quantize &= name.find("shortconv.conv.weight") == std::string::npos;
 
+    // do not quantize Qwen4-Exp's state-space gains or its n-gram conv kernel. They are small and
+    // structural rather than arithmetic: the gains set the decay of the recurrence. A 3 to 4 bit
+    // copy of a tensor in this class leaves a model that loads, runs at full speed and answers
+    // every prompt with the same text, because its input never reaches the residual. ple_conv1d
+    // has a 4-element row, which no block-quantized type can represent, so there is no floor to
+    // set for it in any case. The hyper-connection injection matrices are the same class but do
+    // carry 8 bits, so they get a floor in llama_tensor_get_type instead of being excluded here.
+    quantize &= name.find("ssm_alpha.weight")  == std::string::npos;
+    quantize &= name.find("ssm_beta.weight")   == std::string::npos;
+    quantize &= name.find("ple_conv1d.weight") == std::string::npos;
+
     // do not quantize MiniMax's indexer projection weights, they are tiny
     quantize &= name.find("indexer.k_proj.weight") == std::string::npos;
     quantize &= name.find("indexer.q_proj.weight") == std::string::npos;
@@ -764,6 +775,24 @@ static ggml_type llama_tensor_get_type(quantize_state_impl & qs, const llama_mod
             new_type = llama_tensor_get_type_impl(qs, new_type, tensor, params->ftype, tm.category);
         }
 
+        // Qwen4-Exp's hyper-connection injection matrices decide how the token embedding enters
+        // each layer. At 3 to 4 bits the model answers every prompt with the same text, because
+        // its input never reaches the residual. Eight bits measures clean: the published MTP head
+        // carries both at Q8_0 and drafts at 86% acceptance. So set a floor rather than refusing
+        // to quantize them at all. The shape fallback below still corrects the type if the row
+        // cannot hold it.
+        {
+            const std::string tname(tensor->name);
+            if (tname.find("hc_attn_inject.weight") != std::string::npos ||
+                tname.find("hc_ffn_inject.weight")  != std::string::npos) {
+                const int64_t blck = ggml_blck_size(new_type);
+                const float   bpw  = blck > 0 ? 8.0f * ggml_type_size(new_type) / blck : 32.0f;
+                if (bpw < 8.0f) {
+                    new_type = GGML_TYPE_Q8_0;
+                }
+            }
+        }
+
         // incompatible tensor shapes are handled here - fallback to a compatible type
         new_type = tensor_type_fallback(qs, tensor, new_type);
     }

src/models/qwen4exp.cpp

@@ -266,9 +266,9 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
         layer.nextn.hnorm   = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM,   "weight", il), { hc_dim }, flags);
         layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, flags);
 
-        // unused: graph_mtp reuses the trunk's own model.hc_head_* (see qwen4exp.cpp's
-        // graph_mtp). Kept optional here only so files that still carry this tensor
-        // (e.g. blk.N.nextn.hc_head_* from an older PR 27836-style export) still load.
+        // graph_mtp prefers these when present and falls back to the trunk's own
+        // model.hc_head_*. A draft-only export has no trunk, so this is the only mixer it
+        // has; a full export normally shares the trunk's and leaves these absent.
         layer.nextn.hc_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_HC_HEAD_NORM, "weight", il), { hc_dim }, flags | TENSOR_NOT_REQUIRED);
         layer.nextn.hc_head_down = create_tensor(tn(LLM_TENSOR_NEXTN_HC_HEAD_DOWN, "weight", il), { hc_dim, hc_lr }, flags | TENSOR_NOT_REQUIRED);
         layer.nextn.hc_head_up   = create_tensor(tn(LLM_TENSOR_NEXTN_HC_HEAD_UP,   "weight", il), { hc_lr, hc_dim }, flags | TENSOR_NOT_REQUIRED);
@@ -521,10 +521,16 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_
     GGML_ASSERT(layer.nextn.eh_proj     && "MTP block missing nextn.eh_proj");
     GGML_ASSERT(layer.nextn.enorm       && "MTP block missing nextn.enorm");
     GGML_ASSERT(layer.nextn.hnorm       && "MTP block missing nextn.hnorm");
-    // the MTP head's final mixer is the trunk's own output_hc_* (model.hc_head_*), not a
-    // private per-layer copy: upstream trains one hc mixer, shared between the trunk's last
-    // layer and the draft head, same as the trunk's own final-output call below.
-    GGML_ASSERT(model.hc_head_norm && "QWEN4EXP MTP: model missing hc_head_norm (trunk output mixer)");
+    // The MTP head's final mixer is normally the trunk's own output_hc_* (model.hc_head_*):
+    // upstream trains one hc mixer, shared between the trunk's last layer and the draft head.
+    // A draft-only export has no trunk to share with, so it carries its own
+    // blk.N.nextn.hc_head_* instead. Prefer that when it is there, the same way the LM head
+    // is chosen further down. Without this an MTP-only file loads (trunk_flags makes the
+    // trunk mixer optional for it) and then aborts here.
+    ggml_tensor * mtp_hc_norm = layer.nextn.hc_head_norm ? layer.nextn.hc_head_norm : model.hc_head_norm;
+    ggml_tensor * mtp_hc_down = layer.nextn.hc_head_down ? layer.nextn.hc_head_down : model.hc_head_down;
+    ggml_tensor * mtp_hc_up   = layer.nextn.hc_head_up   ? layer.nextn.hc_head_up   : model.hc_head_up;
+    GGML_ASSERT(mtp_hc_norm && "QWEN4EXP MTP: no output mixer (neither nextn.hc_head_norm nor model.hc_head_norm)");
 
     int sections[4];
     std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
@@ -674,10 +680,10 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_
         res_hc = ggml_reshape_3d(ctx0, res_hc, n_embd, hc, res_hc->ne[1]);
     }
 
-    // the final mixer is shared with the trunk's own output mixer (model.hc_head_*), not a
-    // private per-layer copy -- see the GGML_ASSERT above
+    // the trunk's own output mixer, or the head's private copy for a draft-only export
+    // -- chosen above
     cur = build_hc_mix(res_hc,
-            model.hc_head_norm, model.hc_head_down, model.hc_head_up,
+            mtp_hc_norm, mtp_hc_down, mtp_hc_up,
             nullptr, nullptr, -1);
     cb(cur, "mtp_hc_head", -1);