Skip to content

sd: fix gemma3 vocab and merge filenames and add functions for gemma2 #2249

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
LostRuins merged 2 commits into from
Jun 4, 2026
Merged

sd: fix gemma3 vocab and merge filenames and add functions for gemma2 #2249

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
beab422
sd: simplify vocab and merges loading
wbruna Jun 3, 2026
39e6ce5
sd: fix gemma3 vocab and merge filenames and add functions for gemma2
wbruna Jun 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
0 embd_res/gemma2_merges_utf8_c_str.embd → embd_res/gemma_merges_utf8_c_str.embd
View file
Open in desktop
File renamed without changes.
0 embd_res/gemma2_vocab_json.embd → embd_res/gemma_vocab_json.embd
View file
Open in desktop
File renamed without changes.
109 changes: 39 additions & 70 deletions otherarch/sdcpp/sdtype_adapter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,105 +203,74 @@ static std::string read_str_from_disk(std::string filepath)
return output;
}

std::string load_clip_merges()
static std::string load_embd_file(std::string& cache, const char* filename)
{
static std::string mergesstr; // cached string
if (!mergesstr.empty()) {
return mergesstr; // already loaded
if (cache.empty()) {
std::string filepath = executable_path + filename;
cache = read_str_from_disk(filepath);
}
std::string filepath = executable_path + "embd_res/merges_utf8_c_str.embd";
mergesstr = read_str_from_disk(filepath);
return mergesstr;
return cache;
}

std::string load_clip_merges()
{
static std::string cache;
return load_embd_file(cache, "embd_res/merges_utf8_c_str.embd");
}
std::string load_qwen2_merges()
{
static std::string qwenmergesstr; // cached string
if (!qwenmergesstr.empty()) {
return qwenmergesstr; // already loaded
}
std::string filepath = executable_path + "embd_res/qwen2_merges_utf8_c_str.embd";
qwenmergesstr = read_str_from_disk(filepath);
return qwenmergesstr;
static std::string cache;
return load_embd_file(cache, "embd_res/qwen2_merges_utf8_c_str.embd");
}
std::string load_gemma_merges()
{
static std::string gemmamergesstr; // cached string
if (!gemmamergesstr.empty()) {
return gemmamergesstr; // already loaded
}
std::string filepath = executable_path + "embd_res/gemma2_merges_utf8_c_str.embd";
gemmamergesstr = read_str_from_disk(filepath);
return gemmamergesstr;
static std::string cache;
return load_embd_file(cache, "embd_res/gemma_merges_utf8_c_str.embd");
}
std::string load_gemma_vocab_json()
{
static std::string gemmavocabstr; // cached string
if (!gemmavocabstr.empty()) {
return gemmavocabstr; // already loaded
}
std::string filepath = executable_path + "embd_res/gemma2_vocab_json.embd";
gemmavocabstr = read_str_from_disk(filepath);
return gemmavocabstr;
static std::string cache;
return load_embd_file(cache, "embd_res/gemma_vocab_json.embd");
}
std::string load_gemma2_merges()
{
static std::string cache;
return load_embd_file(cache, "embd_res/gemma2_merges_utf8_c_str.embd");
}
std::string load_gemma2_vocab_json()
{
static std::string cache;
return load_embd_file(cache, "embd_res/gemma2_vocab_json.embd");
}
std::string load_mistral_merges()
{
static std::string mistralmergesstr; // cached string
if (!mistralmergesstr.empty()) {
return mistralmergesstr; // already loaded
}
std::string filepath = executable_path + "embd_res/mistral2_merges_utf8_c_str.embd";
mistralmergesstr = read_str_from_disk(filepath);
return mistralmergesstr;
static std::string cache;
return load_embd_file(cache, "embd_res/mistral2_merges_utf8_c_str.embd");
}
std::string load_mistral_vocab_json()
{
static std::string mistralvocabstr; // cached string
if (!mistralvocabstr.empty()) {
return mistralvocabstr; // already loaded
}
std::string filepath = executable_path + "embd_res/mistral2_vocab_json.embd";
mistralvocabstr = read_str_from_disk(filepath);
return mistralvocabstr;
static std::string cache;
return load_embd_file(cache, "embd_res/mistral2_vocab_json.embd");
}
std::string load_t5_tokenizer_json()
{
static std::string t5str = "";
if (!t5str.empty()) {
return t5str; // already loaded
}
std::string filepath = executable_path + "embd_res/t5_tokenizer_json.embd";
t5str = read_str_from_disk(filepath);
return t5str;
static std::string cache;
return load_embd_file(cache, "embd_res/t5_tokenizer_json.embd");
}
std::string load_umt5_tokenizer_json()
{
static std::string umt5str = "";
if (!umt5str.empty()) {
return umt5str; // already loaded
}
std::string filepath = executable_path + "embd_res/umt5_tokenizer_json.embd";
umt5str = read_str_from_disk(filepath);
return umt5str;
static std::string cache;
return load_embd_file(cache, "embd_res/umt5_tokenizer_json.embd");
}
std::string load_gpt_oss_merges()
{
static std::string mergesstr; // cached string
if (!mergesstr.empty()) {
return mergesstr; // already loaded
}
std::string filepath = executable_path + "embd_res/gpt_oss_merges_utf8_c_str.embd";
mergesstr = read_str_from_disk(filepath);
return mergesstr;
static std::string cache;
return load_embd_file(cache, "embd_res/gpt_oss_merges_utf8_c_str.embd");
}
std::string load_gpt_oss_vocab_json()
{
static std::string vocabstr; // cached string
if (!vocabstr.empty()) {
return vocabstr; // already loaded
}
std::string filepath = executable_path + "embd_res/gpt_oss_vocab_json.embd";
vocabstr = read_str_from_disk(filepath);
return vocabstr;
static std::string cache;
return load_embd_file(cache, "embd_res/gpt_oss_vocab_json.embd");
}

static std::string get_device_override(int value, const char * module = nullptr)
Expand Down