From 78e354899371d65e85e9430e3f5d3b7c14de5e2d Mon Sep 17 00:00:00 2001 From: David Rhodus Date: Fri, 10 Jul 2026 17:32:46 -0700 Subject: [PATCH 1/2] hi-cuda: apply cargo fmt to clear the rustfmt drift blocking CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recent hi-cuda work (commit 09fbc0e / 3fc2ca8) landed with a few method-chain and array-literal expressions that stable rustfmt wants wrapped, so `cargo fmt --all --check` — the blocking `fmt · test` CI job — fails on main. Pure `cargo fmt` normalization; no logic change. Co-Authored-By: Claude Opus 4.8 --- crates/hi-cuda/src/gpu.rs | 19 ++++++++----------- crates/hi-cuda/src/lib.rs | 23 +++++++++++++++++------ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/crates/hi-cuda/src/gpu.rs b/crates/hi-cuda/src/gpu.rs index 047b53b..3b5875d 100644 --- a/crates/hi-cuda/src/gpu.rs +++ b/crates/hi-cuda/src/gpu.rs @@ -3440,15 +3440,13 @@ mod native { // (Q6_K ~0.66 vs f16 2 bytes/param) and wins outright. So in the hybrid plan // (which is chosen precisely when we're not VRAM-starved) skip the f16 cache // for the head and let the dp4a GEMV below handle it. - let cache_output_head = is_output_head - && !self.cache_layer_f16 - && { - let f16_bytes = weight_elements.saturating_mul(std::mem::size_of::()); - self.dequant_f16_cache.borrow().contains_key(matrix_name) - || crate::runtime::free_memory_bytes() - .map(|free| f16_bytes.saturating_mul(2) <= free) - .unwrap_or(true) - }; + let cache_output_head = is_output_head && !self.cache_layer_f16 && { + let f16_bytes = weight_elements.saturating_mul(std::mem::size_of::()); + self.dequant_f16_cache.borrow().contains_key(matrix_name) + || crate::runtime::free_memory_bytes() + .map(|free| f16_bytes.saturating_mul(2) <= free) + .unwrap_or(true) + }; // Fused Q6_K GEMV (M=1 decode): read Q6_K directly instead of dequantizing the // whole matrix to f32 every token — the per-op dequant is ~12x slower for Q6_K // models kept quantized (f16 doesn't fit). Also handles the Q6_K output head in @@ -7268,8 +7266,7 @@ mod native { if token_ids.is_empty() { bail!("append_tokens_logits requires at least one token"); } - if self.config.recurrent_ssm_tensor_layout || !self.supports_batched_text_generation() - { + if self.config.recurrent_ssm_tensor_layout || !self.supports_batched_text_generation() { bail!("append_tokens_logits requires a text decoder layout"); } let dims = self.qwen_dims()?; diff --git a/crates/hi-cuda/src/lib.rs b/crates/hi-cuda/src/lib.rs index fadfea6..f75e5b4 100644 --- a/crates/hi-cuda/src/lib.rs +++ b/crates/hi-cuda/src/lib.rs @@ -4733,7 +4733,10 @@ fn process_continuous_decode_chunk( .max_tokens .saturating_sub(active[idx].generated_tokens.len()); let draft_cap = max_draft.min(remaining.saturating_sub(1)); - let extended = active[idx].context_len().saturating_add(draft_cap).saturating_add(1); + let extended = active[idx] + .context_len() + .saturating_add(draft_cap) + .saturating_add(1); let page_table = match active[idx].page_table_for_token_count(extended, state.kv_page_size) { Ok(pt) => pt, @@ -4759,9 +4762,10 @@ fn process_continuous_decode_chunk( stats .decode_tokens .fetch_add(usize_to_u64(tokens.len()), Ordering::Relaxed); - stats - .decode_micros - .fetch_add(usize_to_u64(elapsed.as_micros() as usize), Ordering::Relaxed); + stats.decode_micros.fetch_add( + usize_to_u64(elapsed.as_micros() as usize), + Ordering::Relaxed, + ); for token in tokens { if retire[idx].is_some() { break; @@ -11861,13 +11865,20 @@ mod tests { // Every emitted token is the model's greedy token, so speculative decode must // reproduce plain greedy exactly, regardless of what the n-gram proposer guesses. - for input in [vec![0u32], vec![1, 0, 1, 0, 1], vec![0, 1, 2, 0, 1, 2, 0, 1]] { + for input in [ + vec![0u32], + vec![1, 0, 1, 0, 1], + vec![0, 1, 2, 0, 1, 2, 0, 1], + ] { let max = 12usize; let greedy = model.generate_greedy_tokens(&input, max, None).unwrap(); let spec = model .generate_greedy_speculative_ngram(&input, max, None, 2, 4) .unwrap(); - assert_eq!(spec.tokens, greedy, "speculative output diverged from greedy"); + assert_eq!( + spec.tokens, greedy, + "speculative output diverged from greedy" + ); // One forward pass yields at least one token, never more tokens than passes*(draft+1). assert!(spec.forward_passes >= 1); assert!(spec.forward_passes <= spec.tokens.len().max(1)); From fc977d9c75941ffc9ea8efb83c3b4ffe12722532 Mon Sep 17 00:00:00 2001 From: David Rhodus Date: Fri, 10 Jul 2026 17:41:18 -0700 Subject: [PATCH 2/2] hi-cuda: add missing non-native-cuda stub for speculative paged decode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recent speculative-decode work added `speculative_greedy_next_tokens_paged_with_page_table` (and a call site in `process_continuous_decode_chunk`) to the `#[cfg(feature = "native-cuda")]` impl, but not the matching stub in the `#[cfg(not(feature = "native-cuda"))]` impl every sibling method has. So the default-feature build — which CI's `cargo test --workspace --exclude hi-mlx` uses — failed to compile with E0599. Add the stub (bails with the standard "built without native-cuda support" message), matching the call site's signature. No native-cuda path change. Co-Authored-By: Claude Opus 4.8 --- crates/hi-cuda/src/gpu.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/hi-cuda/src/gpu.rs b/crates/hi-cuda/src/gpu.rs index 3b5875d..b1edc91 100644 --- a/crates/hi-cuda/src/gpu.rs +++ b/crates/hi-cuda/src/gpu.rs @@ -22855,6 +22855,21 @@ mod non_native { ) } + #[allow(clippy::too_many_arguments)] + pub fn speculative_greedy_next_tokens_paged_with_page_table( + &self, + _sequence: &[u32], + _page_size: usize, + _page_table: &[usize], + _physical_page_count: usize, + _ngram: usize, + _max_draft: usize, + ) -> Result> { + bail!( + "hi-cuda was built without native-cuda support; GPU Qwen paged generation is unavailable" + ) + } + #[allow(clippy::too_many_arguments)] pub fn decode_sampled_next_tokens_batch_paged_with_page_tables( &self,