From 071dbf97f8d91095469dbb22a4354ee25bd5ae21 Mon Sep 17 00:00:00 2001 From: Trey Pendragon Date: Fri, 27 Jun 2025 16:48:49 -0700 Subject: [PATCH 1/2] Add last token pooling support for ORT. I really wanted to a quantized Qwen3 model, this let me do it. --- backends/ort/src/lib.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backends/ort/src/lib.rs b/backends/ort/src/lib.rs index 4e6cea44..5897679f 100644 --- a/backends/ort/src/lib.rs +++ b/backends/ort/src/lib.rs @@ -31,7 +31,7 @@ impl OrtBackend { let pool = match model_type { ModelType::Classifier => Pool::Cls, ModelType::Embedding(pool) => match pool { - Pool::Splade | Pool::LastToken => { + Pool::Splade => { return Err(BackendError::Start(format!( "Pooling {pool} is not supported for this backend. Use `candle` backend instead." ))); @@ -205,7 +205,10 @@ impl Backend for OrtBackend { // CLS pooling Pool::Cls => outputs.slice(s![.., 0, ..]).into_owned().into_dyn(), // Last token pooling is not supported for this model - Pool::LastToken => unreachable!(), + Pool::LastToken => { + let axis_len = outputs.len_of(Axis(1)); + outputs.slice(s![.., axis_len - 1, ..]).into_owned().into_dyn() + }, // Mean pooling Pool::Mean => { if masking { From 5d182018257560e0b6a08a6725872ac6f8b973a0 Mon Sep 17 00:00:00 2001 From: Alvaro Bartolome <36760800+alvarobartt@users.noreply.github.com> Date: Mon, 30 Jun 2025 12:51:05 +0200 Subject: [PATCH 2/2] Update backends/ort/src/lib.rs --- backends/ort/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/backends/ort/src/lib.rs b/backends/ort/src/lib.rs index 5897679f..d75841f1 100644 --- a/backends/ort/src/lib.rs +++ b/backends/ort/src/lib.rs @@ -204,7 +204,6 @@ impl Backend for OrtBackend { let pooled_embeddings = match self.pool { // CLS pooling Pool::Cls => outputs.slice(s![.., 0, ..]).into_owned().into_dyn(), - // Last token pooling is not supported for this model Pool::LastToken => { let axis_len = outputs.len_of(Axis(1)); outputs.slice(s![.., axis_len - 1, ..]).into_owned().into_dyn()