From 3a716b4dae545c3db307594fbc509a95d3e21b6e Mon Sep 17 00:00:00 2001 From: goerch Date: Sat, 7 Oct 2023 06:57:01 +0200 Subject: [PATCH] Fix for #3454 (#3455) Fix: `sentencepiece` tokenizers with added tokens failed with an incorrect assertion --- llama.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index 79ea2b235602e..89dc403b90847 100644 --- a/llama.cpp +++ b/llama.cpp @@ -8200,7 +8200,9 @@ int llama_token_to_piece(const struct llama_model * model, llama_token token, ch buf[0] = llama_token_to_byte(model->vocab, token); return 1; } else { - GGML_ASSERT(false); + // TODO: for now we accept all unsupported token types, + // suppressing them like CONTROL tokens. + // GGML_ASSERT(false); } break; } @@ -8216,7 +8218,9 @@ int llama_token_to_piece(const struct llama_model * model, llama_token token, ch } else if (llama_is_control_token(model->vocab, token)) { ; } else { - GGML_ASSERT(false); + // TODO: for now we accept all unsupported token types, + // suppressing them like CONTROL tokens. + // GGML_ASSERT(false); } break; }