From 9ada2b4408924fe1b7c0ac8f7c7abf3dcb7f1f51 Mon Sep 17 00:00:00 2001 From: Kyle Mistele Date: Fri, 1 Nov 2024 01:29:47 -0500 Subject: [PATCH] fix: hermes tool parser None checks & formatting Signed-off-by: Kyle Mistele --- .../openai/tool_parsers/hermes_tool_parser.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py index faa6f653b835c..a385cd97af206 100644 --- a/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py @@ -213,6 +213,7 @@ def extract_tool_calls_streaming( try: + # try to load the tool call if it's present current_tool_call = partial_json_parser.loads( tool_call_portion or "{}", flags) if tool_call_portion else None @@ -221,6 +222,11 @@ def extract_tool_calls_streaming( logger.debug('not enough tokens to parse into JSON yet') return None + if not current_tool_call: + logger.debug( + 'current_tool_call is none. waiting for more tokens') + return None + # case - we haven't sent the tool name yet. If it's available, send # it. otherwise, wait until it's available. if not self.current_tool_name_sent: @@ -260,8 +266,14 @@ def extract_tool_calls_streaming( # main logic for tool parsing here - compare prev. partially-parsed # JSON to the current partially-parsed JSON - prev_arguments = ( - self.prev_tool_call_arr[self.current_tool_id].get("arguments")) + prev_call_state = self.prev_tool_call_arr[self.current_tool_id] + + # get the previous and current versions of the arguments for the + # call if they exist. + if prev_call_state: + prev_arguments = prev_call_state.get('arguments') + else: + prev_arguments = None cur_arguments = current_tool_call.get("arguments") logger.debug("diffing old arguments: %s", prev_arguments)