From 913fe8be04e5f596517468eb7754b9e5ed533189 Mon Sep 17 00:00:00 2001 From: PAB Date: Tue, 1 Aug 2023 15:33:36 +0200 Subject: [PATCH] COSMIT Move tokenization in its own function (#32) --- bark.cpp | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/bark.cpp b/bark.cpp index f4bd34d..bf0a0a1 100644 --- a/bark.cpp +++ b/bark.cpp @@ -1227,13 +1227,11 @@ bark_vocab::id gpt_sample( return gpt_multinomial_sample(logits, rng, temp, eos_p); } -bark_sequence bark_tokenize_input( - const char * text, - const bark_vocab & vocab, - const int32_t block_size) { +bark_sequence bark_tokenize_input(const char * text, const bark_vocab & vocab, int32_t block_size) { + // max bark length: 256 int32_t max_ctx_size = std::min(block_size, 256); - int32_t n_tokens; + bark_sequence tokens(max_ctx_size); bert_tokenize(vocab, text, tokens.data(), &n_tokens, max_ctx_size); @@ -1494,8 +1492,6 @@ bool bark_generate_audio( const bark_vocab& vocab, const char * text, const int n_threads) { - bark_sequence tokens; - // TODO move into params // const int top_k = 10; const int seed = 0; @@ -1513,8 +1509,8 @@ bool bark_generate_audio( std::mt19937 rng(seed); - // bert tokenizer - const int32_t block_size = model.text_model.hparams.block_size; + // tokenize input (bert tokenizer) + int32_t block_size = model.text_model.hparams.block_size; bark_sequence tokens = bark_tokenize_input(text, vocab, block_size); printf("%s: prompt: '%s'\n", __func__, text);