From 41dddb7dfbc1acd96c43ba80de2ce37a3849d66f Mon Sep 17 00:00:00 2001
From: James Lal <james@littlebearlabs.io>
Date: Wed, 12 Mar 2025 01:38:01 -0600
Subject: [PATCH] chore: Upgrade to tokenizers 0.21.0

---
 Cargo.lock        | 6 +++---
 Cargo.toml        | 2 +-
 router/src/lib.rs | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 73cf8f8a..324f9a73 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "accelerate-src"
@@ -4488,9 +4488,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokenizers"
-version = "0.19.1"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e500fad1dd3af3d626327e6a3fe5050e664a6eaa4708b8ca92f1794aaf73e6fd"
+checksum = "9ecededfed68a69bc657e486510089e255e53c3d38cc7d4d59c8742668ca2cae"
 dependencies = [
  "aho-corasick",
  "derive_builder",
diff --git a/Cargo.toml b/Cargo.toml
index b979d86c..f44df03c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -24,7 +24,7 @@ hf-hub = { version = "0.3.2", features = ["tokio", "online"], default-features =
 metrics = "0.23"
 nohash-hasher = "0.2"
 num_cpus = "1.16.0"
-tokenizers = { version = "0.19.1", default-features = false, features = ["onig", "esaxx_fast"] }
+tokenizers = { version = "0.21.0", default-features = false, features = ["onig", "esaxx_fast"] }
 tokio = { version = "1.25", features = ["rt", "rt-multi-thread", "parking_lot", "sync", "signal"] }
 tracing = "0.1"
 serde = { version = "1.0", features = ["serde_derive"] }
diff --git a/router/src/lib.rs b/router/src/lib.rs
index b4900b32..76cebccc 100644
--- a/router/src/lib.rs
+++ b/router/src/lib.rs
@@ -147,13 +147,13 @@ pub async fn run(
             .build()
             .unwrap();
         match tokenizer.get_post_processor() {
-            None => tokenizer.with_post_processor(template),
+            None => tokenizer.with_post_processor(Some(template)),
             Some(post_processor) => {
                 let post_processor = Sequence::new(vec![
                     post_processor.clone(),
                     PostProcessorWrapper::Template(template),
                 ]);
-                tokenizer.with_post_processor(post_processor)
+                tokenizer.with_post_processor(Some(post_processor))
             }
         };
     }