WIP: Intel GPU / SYCL (proof-of-concept)

Cargo.toml

@@ -14,3 +14,29 @@ candle-nn = "0.11"
 candle-transformers = "0.11"
 tokenizers = { version = "0.23", default-features = false, features = ["onig"] }
 fst = "0.4"
+
+# SYCL (Intel oneAPI) backend — proof-of-concept.
+#
+# candle 0.11 on crates.io has no `sycl` feature, so the `sycl` Cargo feature in
+# `crane-core`/`crane`/`crane-serve` needs a candle build that provides one. This
+# fork is candle 0.11.0 plus an off-by-default `sycl` feature (adds the
+# `candle-sycl-kernels` crate, excluded from the workspace like the CUDA kernels,
+# so a plain build never needs the oneAPI compiler).
+#
+# The patch is global — it can't be gated on the `sycl` feature — but with `sycl`
+# off the fork is stock candle 0.11.0, so CPU/CUDA/Metal builds are unaffected.
+# This is temporary POC wiring and is not meant to land on `main`; drop it once
+# the SYCL backend is upstreamed.
+#
+# rev c91b22a = "candle SYCL support": candle 0.11.0 + the off-by-default `sycl`
+# feature + (a) the matmul densify fix for grouped-query attention and (b) the
+# out-of-tree launch surface (`SyclStorage::{buf,elems,from_buffer}`,
+# `SyclDevice::alloc_bytes`, `Queue::native_ptr`) + native `sigmoid`.
+# For local candle iteration, swap to a path dep:
+#   candle-core        = { path = "../candle/candle-core" }
+#   candle-nn          = { path = "../candle/candle-nn" }
+#   candle-transformers = { path = "../candle/candle-transformers" }
+[patch.crates-io]
+candle-core = { git = "https://github.com/Hahihula/candle", rev = "c91b22a54b8d2a9046db81061836dee08140328c" }
+candle-nn = { git = "https://github.com/Hahihula/candle", rev = "c91b22a54b8d2a9046db81061836dee08140328c" }
+candle-transformers = { git = "https://github.com/Hahihula/candle", rev = "c91b22a54b8d2a9046db81061836dee08140328c" }

README.md

@@ -289,6 +289,43 @@ Current limitations:
   prompt lengths from one process makes exhaustion likelier, because the caching allocator
   retains a bucket set per shape it has seen.
 
+#### Intel GPU / SYCL (proof-of-concept)
+
+SYCL/oneAPI support targets Intel GPUs. candle 0.11 on crates.io has no `sycl`
+feature, so the root `Cargo.toml` `[patch.crates-io]` points
+`candle-core`/`candle-nn`/`candle-transformers` at
+[`Hahihula/candle`](https://github.com/Hahihula/candle) — candle 0.11.0 plus an
+**off-by-default** `sycl` feature (native rms_norm / softmax / rope / sigmoid
+kernels + a small `pub` launch surface). With `sycl` off that fork is stock
+candle, so CPU/CUDA/Metal builds are unchanged. POC only, not for `main`.
+
+Needs the Intel oneAPI toolchain (`icpx`, oneMKL) and the Level-Zero GPU runtime;
+the `intel/oneapi-basekit` image bundles both, and `--device /dev/dri` exposes an
+Intel GPU (`sycl-ls` should list a `level_zero:gpu` entry). `--features sycl`
+selects `DeviceConfig::Sycl(0)` in the examples / `crane-serve` device ladder
+(`Device::new_sycl`) and builds `crane-core/kernels/sycl/gdn.cpp` into
+`libcrane_gdn_sycl.so` — a **fused Gated Delta Net recurrence** kernel, the SYCL
+counterpart of `kernels/cuda/gdn.cu`.
+
+```bash
+source /opt/intel/oneapi/setvars.sh          # skip inside the oneAPI container
+cargo build --release --features sycl
+cargo run  --release --features sycl -p crane-examples --bin chat_cli -- \
+    -m /path/to/Qwen3.5-0.8B
+```
+
+`libcandle_sycl.so` / `libcrane_gdn_sycl.so` are linked by rpath into an rlib and
+that rpath doesn't reach the final binary, so add their `OUT_DIR`s to
+`LD_LIBRARY_PATH` when running (do not clobber the image's oneAPI paths). A
+one-shot containerised build/run/test recipe that handles all of this lives in
+[`contrib/sycl/`](contrib/sycl/).
+
+Verified on an Intel Arc iGPU (Meteor Lake): `Qwen3-0.6B` (dense) and
+**`Qwen3.5-0.8B`** (hybrid GDN + attention) generate coherent text. The fused GDN
+kernel matches the portable reference (`cos = 1.0`, `--test sycl_kernels`) and is
+~15% faster at decode (3.8 → 4.4 tok/s); `CRANE_GDN_PORTABLE=1` forces the
+op-by-op path. It's a naive v0 (no shared-memory staging), so there is headroom.
+
 ### OpenAI API Server
 
 Start a server compatible with OpenAI SDK and SGLang client:

contrib/sycl/Dockerfile

@@ -0,0 +1,22 @@
+# Crane SYCL (Intel oneAPI) build + run environment — proof-of-concept.
+#
+# Base image ships the icpx SYCL compiler, oneMKL, and the Level-Zero GPU
+# runtime, so an Intel GPU passed in with `--device /dev/dri` is usable out of
+# the box (`sycl-ls` lists a `level_zero:gpu` entry).
+FROM intel/oneapi-basekit:2025.1.0-0-devel-ubuntu24.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        curl ca-certificates build-essential pkg-config git \
+        libssl-dev cmake libasound2-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Rust (edition 2024 needs >= 1.85; pin to match the host stable toolchain).
+ENV RUSTUP_HOME=/opt/rustup \
+    CARGO_HOME=/opt/cargo \
+    PATH=/opt/cargo/bin:$PATH
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
+      | sh -s -- -y --profile minimal --default-toolchain 1.93.0 \
+    && rustc --version
+
+WORKDIR /src

contrib/sycl/README.md

@@ -0,0 +1,79 @@
+# Crane on Intel SYCL / oneAPI (proof-of-concept)
+
+candle 0.11 on crates.io has no SYCL backend, so the root `Cargo.toml`
+`[patch.crates-io]` pins the three `candle-*` crates to
+[`Hahihula/candle`](https://github.com/Hahihula/candle) rev `c91b22a` — candle
+0.11.0 plus an **off-by-default** `sycl` feature. With `sycl` off it is stock
+candle, so CPU/CUDA/Metal builds are unchanged. Not for `main`.
+
+The fork adds, behind `sycl`:
+
+- native SYCL kernels for `candle-nn`'s fused ops — `rms_norm`, `softmax`,
+  `rope`, `sigmoid` (no CPU round-trip);
+- a matmul "densify non-contiguous operands" fix (grouped-query attention);
+- a small `pub` launch surface — `SyclStorage::{buf, elems, from_buffer}`,
+  `SyclDevice::alloc_bytes`, `Queue::native_ptr()` — so an out-of-tree kernel can
+  submit onto candle's in-order queue (`candle-sycl-matmul-fix.patch` here is the
+  older standalone form of the matmul fix).
+
+## What's wired in Crane
+
+- `sycl` feature on `crane-core` / `crane` / `crane-serve` / `example` →
+  `candle-*/sycl`.
+- `DeviceConfig::Sycl(u32)` + `Device::new_sycl` in every device selector; the
+  examples / `crane-serve` device ladder pick `Sycl(0)` when built `--features sycl`.
+- **Fused Gated Delta Net recurrence** — `crane-core/kernels/sycl/gdn.cpp` (icpx →
+  `libcrane_gdn_sycl.so` via `crane-core/build.rs`) + `ops/gdn/sycl_backend.rs`
+  launcher, dispatched by `apply_recurrence` for `is_sycl()`. The SYCL
+  counterpart of `kernels/cuda/gdn.cu`. `CRANE_GDN_PORTABLE=1` forces the
+  op-by-op path.
+
+## Requirements
+
+The Intel oneAPI toolchain (`icpx`, oneMKL) plus the Level-Zero GPU runtime. The
+`intel/oneapi-basekit` image has all of it; with `--device /dev/dri` an Intel GPU
+is visible out of the box (`sycl-ls` lists a `level_zero:gpu` entry). The fork's
+SYCL runtime **only enumerates GPU devices**, so a usable Intel GPU is required.
+
+## One-shot container recipe
+
+```bash
+# build the image (oneAPI basekit + Rust + libssl/alsa/cmake)
+docker build -t crane-sycl:dev -f contrib/sycl/Dockerfile contrib/sycl
+
+contrib/sycl/run.sh build              # compile
+contrib/sycl/run.sh test               # cargo test --test sycl_kernels (kernel vs portable)
+CRANE_SYCL_MODELS=/path/to/models \
+  contrib/sycl/run.sh chat -m /models/Qwen3.5-0.8B --max-new-tokens 200
+```
+
+`run.sh <cmd>` runs an arbitrary command in the container. It mounts the repo,
+`~/.cargo/{registry,git}`, and a persistent `../crane-sycl-docker-target/` (kept
+outside the repo — the container writes it as root). It sets `LD_LIBRARY_PATH` to
+the two build.rs `OUT_DIR`s **prepended to** the image's baked oneAPI paths — do
+not clobber those (they carry the split `libmkl_sycl_*` deps and the Level-Zero
+adapter that GPU discovery needs).
+
+## Status
+
+Verified on an Intel Arc iGPU (Meteor Lake), safetensors + F16:
+
+| Model | Arch | Notes |
+|---|---|---|
+| `Qwen3-0.6B` | dense | ~7–8 tok/s |
+| `Qwen3.5-0.8B` | hybrid GDN + attention | fused GDN kernel; ~4.4 tok/s |
+
+Both generate coherent text and exit cleanly. The fused GDN kernel matches the
+portable reference exactly (`cos = 1.0` for K=128/64 × prefill/decode) and is
+~15% faster at decode than the op-by-op path (3.8 → 4.4 tok/s).
+
+### Not yet done
+
+- The GDN kernel is a **naive v0** — no shared-memory staging of `k_t`/`q_t`, and
+  the per-work-item state column (`Scol[128]`) spills to scratch on Intel. SLM
+  tiling of the state is the next optimisation; `kernels/cuda/gdn.cu` is the
+  reference for the math and the 4-way ILP split.
+- GGUF path leaves SYCL on F32 side tensors (`qwen3_5/model.rs` `from_gguf` only
+  gives CUDA/Metal/ROCm F16/BF16); add `|| device.is_sycl()` there for the F16
+  memory win. Untested with a quantized Qwen3.5.
+- Multi-Intel-GPU, `crane-serve` continuous batching, and vision towers untried.

contrib/sycl/candle-sycl-matmul-fix.patch

@@ -0,0 +1,131 @@
+From ce149eea5c98afda1f45665565914f57ce4fde53 Mon Sep 17 00:00:00 2001
+From: hahihula <hahihula@gmail.com>
+Date: Fri, 28 Aug 2026 20:02:17 +0200
+Subject: [PATCH] sycl: densify non-contiguous matmul operands instead of
+ bailing
+
+The oneMKL gemm_batch path only accepted a tightly-packed batch of
+row-major / plain-transpose tiles. Grouped-query attention feeds a
+doubly-strided narrow view of the KV cache, which candle's CPU/CUDA/Metal
+matmul copy to contiguous internally; the SYCL backend bailed with
+"not a contiguous batch of N-element matrices".
+
+mm_operand now returns None for any layout gemm_batch can't express
+directly, and matmul's prep closure materialises those operands into a
+dense row-major copy (the path already used for broadcast operands).
+
+Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
+---
+ candle-core/src/sycl_backend/mod.rs | 63 ++++++++++++++++++-----------
+ 1 file changed, 39 insertions(+), 24 deletions(-)
+
+diff --git a/candle-core/src/sycl_backend/mod.rs b/candle-core/src/sycl_backend/mod.rs
+index fd64495a..2899e041 100644
+--- a/candle-core/src/sycl_backend/mod.rs
++++ b/candle-core/src/sycl_backend/mod.rs
+@@ -953,15 +953,25 @@ impl BackendStorage for SyclStorage {
+         rhs_l: &Layout,
+     ) -> Result<Self> {
+         let dt = self.sd()?;
+-        // oneMKL's strided `gemm_batch` does not honour a zero batch stride, so a
+-        // broadcast operand is materialised into a contiguous `b`-copy first.
++        // oneMKL's strided `gemm_batch` wants one uniform batch stride and a
++        // row-major / plain-transpose 2-D tile. Anything it can't express
++        // directly (a broadcast/zero batch stride, a ragged batch nesting, or a
++        // 2-D tile that is neither row-major nor a plain transpose — e.g. a
++        // doubly-strided narrow view of a KV cache) is first materialised into a
++        // dense row-major `b*rows*cols` copy. `mm_operand` returns `None` for
++        // the stride in exactly those cases.
+         let prep = |buf: &DeviceBuffer,
+                     l: &Layout,
+                     rows: usize,
+                     cols: usize|
+          -> Result<(Option<SyclStorage>, bool, usize)> {
+             let (trans, stride) = mm_operand(l, rows, cols)?;
+-            if stride == 0 && b > 1 {
++            let need_dense = match stride {
++                None => true,
++                Some(0) => b > 1,
++                Some(_) => false,
++            };
++            if need_dense {
+                 let dense = self.device.alloc(self.dtype, b * rows * cols)?;
+                 wrap(k::copy_strided(
+                     &self.device.queue,
+@@ -974,7 +984,7 @@ impl BackendStorage for SyclStorage {
+                 ))?;
+                 Ok((Some(dense), false, rows * cols))
+             } else {
+-                Ok((None, trans, stride))
++                Ok((None, trans, stride.unwrap_or(rows * cols)))
+             }
+         };
+         let (lhs_owned, transa, stride_a) = prep(&self.buffer, lhs_l, m, kk)?;
+@@ -1229,7 +1239,14 @@ impl SyclStorage {
+
+ /// Classify a matmul operand: is it stored transposed relative to row-major, and
+ /// what is its batch stride. `rows`/`cols` are the logical (non-transposed) dims.
+-fn mm_operand(l: &Layout, rows: usize, cols: usize) -> Result<(bool, usize)> {
++/// Describes how a matmul operand's layout maps onto oneMKL `gemm_batch`.
++///
++/// Returns `(trans, Some(batch_stride))` when the layout is usable directly:
++/// `batch_stride == 0` marks a broadcast operand (the caller densifies it when
++/// `b > 1`). Returns `(_, None)` when the layout cannot be expressed as a
++/// row-major / plain-transpose tile with a uniform batch stride and the caller
++/// must materialise a dense row-major copy first.
++fn mm_operand(l: &Layout, rows: usize, cols: usize) -> Result<(bool, Option<usize>)> {
+     let stride = l.stride();
+     let rank = stride.len();
+     if rank < 2 {
+@@ -1244,31 +1261,29 @@ fn mm_operand(l: &Layout, rows: usize, cols: usize) -> Result<(bool, usize)> {
+     } else if (rs == 1 || rows == 1) && (cs == rows || cols == 1) {
+         true
+     } else {
+-        sycl_bail!(
+-            "matmul operand with strides {stride:?} (dims {dims:?}) is neither row-major nor a \
+-             plain transpose"
+-        )
++        // Neither row-major nor a plain transpose (e.g. a doubly-strided narrow
++        // view): caller densifies.
++        return Ok((false, None));
+     };
+     let batch_dims = &dims[..rank - 2];
+     let batch_strides = &stride[..rank - 2];
+     let batch: usize = batch_dims.iter().product();
+     let inner = rows * cols;
+-    let batch_stride = if batch <= 1 || batch_strides.iter().all(|&s| s == 0) {
+-        0 // scalar batch, or a broadcast operand
+-    } else {
+-        let mut expect = inner;
+-        for i in (0..batch_dims.len()).rev() {
+-            if batch_dims[i] != 1 && batch_strides[i] != expect {
+-                sycl_bail!(
+-                    "matmul batch strides {stride:?} (dims {dims:?}) are not a contiguous batch \
+-                     of {inner}-element matrices"
+-                );
+-            }
+-            expect *= batch_dims[i];
++    if batch <= 1 {
++        return Ok((trans, Some(inner)));
++    }
++    if batch_strides.iter().all(|&s| s == 0) {
++        return Ok((trans, Some(0))); // broadcast operand
++    }
++    let mut expect = inner;
++    for i in (0..batch_dims.len()).rev() {
++        if batch_dims[i] != 1 && batch_strides[i] != expect {
++            // Ragged batch nesting: caller densifies.
++            return Ok((false, None));
+         }
+-        inner
+-    };
+-    Ok((trans, batch_stride))
++        expect *= batch_dims[i];
++    }
++    Ok((trans, Some(inner)))
+ }
+
+ impl BackendDevice for SyclDevice {
+-- 
+2.47.3
+

contrib/sycl/run.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Build and run Crane on the Intel SYCL backend inside the oneAPI container.
+#
+#   contrib/sycl/run.sh build
+#   contrib/sycl/run.sh chat  -m /models/<dir>
+#   contrib/sycl/run.sh test          # cargo test --test sycl_kernels
+#   contrib/sycl/run.sh <any bash command run inside the container>
+#
+# Env overrides:
+#   CRANE_SYCL_MODELS  host dir of models, mounted read-only at /models
+#   CRANE_SYCL_TARGET  host dir for the container's CARGO_TARGET_DIR
+#   CRANE_CANDLE       host path of a Hahihula/candle checkout to mount at
+#                      /candle (only needed if the root Cargo.toml [patch] uses
+#                      a `path = "../candle/..."` dep instead of the git rev)
+set -euo pipefail
+
+REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+IMAGE=crane-sycl:dev
+MODELS="${CRANE_SYCL_MODELS:-}"
+# Kept outside the repo: the container writes it as root.
+TARGET="${CRANE_SYCL_TARGET:-$REPO/../crane-sycl-docker-target}"
+CANDLE="${CRANE_CANDLE:-}"
+mkdir -p "$TARGET"
+
+docker image inspect "$IMAGE" >/dev/null 2>&1 || \
+  docker build -t "$IMAGE" -f "$REPO/contrib/sycl/Dockerfile" "$REPO/contrib/sycl"
+
+BUILD='cargo build --release -p crane-examples --bin chat_cli --features sycl'
+# Prepend both build.rs OUT_DIRs (libcandle_sycl.so / libcrane_gdn_sycl.so —
+# their rpath does not reach the final binary) to the image's baked oneAPI
+# LD_LIBRARY_PATH. Do NOT clobber it: it carries mkl / umf / pti / level-zero
+# adapter dirs that both the split libmkl_sycl_* deps and GPU discovery need.
+RUN_PRE='export LD_LIBRARY_PATH="$(find /target -name "libcandle_sycl.so" -o -name "libcrane_gdn_sycl.so" | xargs -rn1 dirname | sort -u | paste -sd:):${LD_LIBRARY_PATH}"'
+case "${1:-build}" in
+  build) CMD="$BUILD" ;;
+  chat)  shift; CMD="$BUILD && $RUN_PRE && /target/release/chat_cli $*" ;;
+  test)  shift; CMD="$RUN_PRE && cargo test -p crane-core --release --features sycl --test sycl_kernels -- --nocapture $*" ;;
+  *)     CMD="$*" ;;
+esac
+
+ARGS=(
+  --rm -i
+  --device /dev/dri:/dev/dri
+  -v "$REPO":/src
+  -v "$TARGET":/target
+  -v "$HOME/.cargo/registry":/opt/cargo/registry
+  -v "$HOME/.cargo/git":/opt/cargo/git
+  -e CARGO_TARGET_DIR=/target
+  -w /src
+)
+[ -n "$MODELS" ] && ARGS+=( -v "$MODELS":/models:ro )
+[ -n "$CANDLE" ] && ARGS+=( -v "$CANDLE":/candle:ro )
+
+exec docker run "${ARGS[@]}" "$IMAGE" bash -lc "$CMD"

crane-core/Cargo.toml

@@ -85,6 +85,13 @@ cudnn = ["candle-core/cudnn"]
 # AMD ROCm/HIP backend (experimental). candle 0.11 on crates.io has no `rocm`
 # feature, so this can't forward to `candle-core/rocm` yet.
 rocm = []
+# Intel SYCL / oneAPI backend (proof-of-concept). Forwards to the `sycl` feature
+# of the candle fork wired in via `[patch.crates-io]` in the root Cargo.toml.
+sycl = [
+    "candle-core/sycl",
+    "candle-nn/sycl",
+    "candle-transformers/sycl",
+]
 mkl = [
     "dep:intel-mkl-src",
     "candle-core/mkl",