Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Re-merge main into dev #185

Merged
merged 4 commits into from
Dec 1, 2024
Merged

Re-merge main into dev #185

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c23405b
Merge pull request #140 from bespokelabsai/dev
CharlieJCJ Nov 19, 2024
6252238
Merge pull request #174 from bespokelabsai/dev
vutrung96 Nov 26, 2024
3a49331
allow special tokens when encoding text for token accounting
RyanMarten Nov 30, 2024
87cfc3d
Merge pull request #181 from bespokelabsai/ryanm/special-token-error
vutrung96 Nov 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions src/bespokelabs/curator/request_processor/openai_online_request_processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -674,7 +674,7 @@ def num_tokens_consumed_from_request(
num_tokens += 4 # every message follows <im_start>{role/name}\n{content}<im_end>\n
for key, value in message.items():
try:
num_tokens += len(encoding.encode(str(value)))
num_tokens += len(encoding.encode(str(value), disallowed_special=()))
except TypeError:
logger.warning(
f"Failed to encode value {value} with tiktoken to count tokens. Instead assuming a token for every 4 characters."
Expand All @@ -688,11 +688,13 @@ def num_tokens_consumed_from_request(
else:
prompt = api_specific_request_json["prompt"]
if isinstance(prompt, str): # single prompt
prompt_tokens = len(encoding.encode(prompt))
prompt_tokens = len(encoding.encode(prompt, disallowed_special=()))
num_tokens = prompt_tokens + completion_tokens
return num_tokens
elif isinstance(prompt, list): # multiple prompts
prompt_tokens = sum([len(encoding.encode(p)) for p in prompt])
prompt_tokens = sum(
[len(encoding.encode(p, disallowed_special=())) for p in prompt]
)
num_tokens = prompt_tokens + completion_tokens * len(prompt)
return num_tokens
else:
Expand All @@ -703,10 +705,10 @@ def num_tokens_consumed_from_request(
elif api_endpoint == "embeddings":
input = api_specific_request_json["input"]
if isinstance(input, str): # single input
num_tokens = len(encoding.encode(input))
num_tokens = len(encoding.encode(input, disallowed_special=()))
return num_tokens
elif isinstance(input, list): # multiple inputs
num_tokens = sum([len(encoding.encode(i)) for i in input])
num_tokens = sum([len(encoding.encode(i, disallowed_special=())) for i in input])
return num_tokens
else:
raise TypeError(
Expand Down