fix: unkown characters in generation streaming (#293)

withcatai · Aug 7, 2024 · 097b3ec · 097b3ec
1 parent a2b2bc3
commit 097b3ec
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/src/evaluator/LlamaChat/LlamaChat.ts b/src/evaluator/LlamaChat/LlamaChat.ts
@@ -2215,7 +2215,7 @@ class GenerateResponseState<const Functions extends ChatModelFunctions | undefin
     }
 
     public waitOnPartialCharactersOrWhiteSpaceTokens() {
-        if (this.currentText === UNKNOWN_UNICODE_CHAR || (
+        if (this.currentText.endsWith(UNKNOWN_UNICODE_CHAR) || (
             (this.grammar?.trimWhitespaceSuffix || this.trimWhitespaceSuffix) && this.currentText?.trim() === ""
         )) {
             if (this.currentQueuedTokenRelease != null)

diff --git a/src/evaluator/LlamaCompletion.ts b/src/evaluator/LlamaCompletion.ts
@@ -662,7 +662,7 @@ export class LlamaCompletion {
                 const text = model.detokenize([token]);
                 const queuedTokenRelease = streamRegulator.addChunk({tokens, text});
 
-                if (text === UNKNOWN_UNICODE_CHAR || (
+                if (text.endsWith(UNKNOWN_UNICODE_CHAR) || (
                     (grammar?.trimWhitespaceSuffix || trimWhitespaceSuffix) && text.trim() === ""
                 )) {
                     locksToReleaseOnValidGeneration.push(queuedTokenRelease.createTextIndexLock(0));