From 69e91b1d7f8f0e45ba1ed83c737512da47c44418 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 May 2026 05:03:47 +0000 Subject: [PATCH] Update tokenizers requirement from 0.22 to 0.23 Updates the requirements on [tokenizers](https://github.com/huggingface/tokenizers) to permit the latest version. - [Release notes](https://github.com/huggingface/tokenizers/releases) - [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md) - [Commits](https://github.com/huggingface/tokenizers/compare/v0.22.0...v0.23.1) --- updated-dependencies: - dependency-name: tokenizers dependency-version: 0.23.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- crates/bpe/benchmarks/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bpe/benchmarks/Cargo.toml b/crates/bpe/benchmarks/Cargo.toml index 6fb1a6a..78ceb96 100644 --- a/crates/bpe/benchmarks/Cargo.toml +++ b/crates/bpe/benchmarks/Cargo.toml @@ -23,4 +23,4 @@ bpe-openai = { path = "../../bpe-openai" } criterion = "0.8" rand = "0.10" tiktoken-rs = "0.9" -tokenizers = { version = "0.22", features = ["http"] } +tokenizers = { version = "0.23", features = ["http"] }