WIP: testing only

bindings/node/Cargo.toml

@@ -13,7 +13,7 @@ crate-type = ["cdylib"]
 napi        = "3"
 napi-derive = "3"
 serde       = { version = "1.0.163", features = ["derive"] }
-tokenizers  = { path = "../../tokenizers/" }
+tokenizers  = { path = "../../tokenizers" }
 ahash = { version = "0.8.11", features = ["serde"] }
 
 [build-dependencies]

tokenizers/Cargo.toml

@@ -4,6 +4,7 @@ authors = [
   "Nicolas Patry <patry.nicolas@protonmail.com>",
 ]
 edition = "2018"
+resolver = "2"
 name = "tokenizers"
 version = "0.23.2-dev.0"
 homepage = "https://github.com/huggingface/tokenizers"
@@ -83,7 +84,7 @@ itertools = "0.14"
 log = "0.4"
 derive_builder = "0.20"
 spm_precompiled = "0.1.3"
-hf-hub = { version = "0.4.1", features = ["ureq"], default-features = false, optional = true }
+hf-hub = { version = "1.0.0-rc.1", features = ["blocking"], optional = true }
 daachorse = "1.0.1"
 paste = "1.0.14"
 macro_rules_attribute = "0.2.0"
@@ -102,7 +103,7 @@ esaxx_fast = ["esaxx-rs/cpp"]
 progressbar = ["indicatif"]
 http = ["hf-hub"]
 unstable_wasm = ["fancy-regex", "getrandom/wasm_js"]
-rustls-tls = ["hf-hub?/rustls-tls"]
+rustls-tls = ["hf-hub/rustls-tls"]
 
 [dev-dependencies]
 criterion = "0.6"

tokenizers/src/utils/from_pretrained.rs

@@ -1,5 +1,5 @@
 use crate::Result;
-use hf_hub::{api::sync::ApiBuilder, Repo, RepoType};
+use hf_hub::HFClient;
 use std::collections::HashMap;
 use std::path::PathBuf;
 
@@ -57,12 +57,30 @@ pub fn from_pretrained<S: AsRef<str>>(
         .into());
     }
 
-    let mut builder = ApiBuilder::from_env();
+    let mut builder = HFClient::builder();
     if let Some(token) = params.token {
-        builder = builder.with_token(Some(token));
+        builder = builder.token(token);
     }
-    let api = builder.build()?;
-    let repo = Repo::with_revision(identifier, RepoType::Model, params.revision);
-    let api = api.repo(repo);
-    Ok(api.get("tokenizer.json")?)
+    if !params.user_agent.is_empty() {
+        let user_agent = params
+            .user_agent
+            .iter()
+            .map(|(k, v)| format!("{k}/{v}"))
+            .collect::<Vec<_>>()
+            .join("; ");
+        builder = builder.user_agent(user_agent);
+    }
+    let client = builder.build_sync()?;
+
+    let (owner, name) = identifier.split_once('/').ok_or_else(|| {
+        format!("Model \"{identifier}\" is not a valid repo id, expected format \"owner/name\"")
+    })?;
+
+    let path = client
+        .model(owner, name)
+        .download_file()
+        .filename("tokenizer.json")
+        .revision(params.revision)
+        .send()?;
+    Ok(path)
 }

tokenizers/tests/from_pretrained.rs

@@ -3,7 +3,7 @@ use tokenizers::{FromPretrainedParameters, Result, Tokenizer};
 
 #[test]
 fn test_from_pretrained() -> Result<()> {
-    let tokenizer = Tokenizer::from_pretrained("bert-base-cased", None)?;
+    let tokenizer = Tokenizer::from_pretrained("google-bert/bert-base-cased", None)?;
     let encoding = tokenizer.encode("Hey there dear friend!", false)?;
     assert_eq!(
         encoding.get_tokens(),
@@ -46,7 +46,7 @@ fn test_from_pretrained_invalid_model() {
 #[test]
 fn test_from_pretrained_invalid_revision() {
     let tokenizer = Tokenizer::from_pretrained(
-        "bert-base-cased",
+        "google-bert/bert-base-cased",
         Some(FromPretrainedParameters {
             revision: "gpt?".to_string(),
             ..Default::default()