Skip to content

Commit

Permalink
Try to Phi3V and Ultravox
Browse files Browse the repository at this point in the history
Signed-off-by: DarkLight1337 <[email protected]>
  • Loading branch information
DarkLight1337 committed Dec 23, 2024
1 parent e30deec commit bbd5997
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 5 deletions.
3 changes: 2 additions & 1 deletion tests/multimodal/test_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -618,7 +618,8 @@ def _test_processing_cache_correctness(
hf_processor_mm_kwargs={},
)

assert baseline_result == cached_result, f"Failed (idx={batch_idx})"
assert baseline_result == cached_result, (
f"Failed ({batch_idx=}, {mm_data=})")


# yapf: disable
Expand Down
18 changes: 14 additions & 4 deletions vllm/multimodal/processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -866,10 +866,6 @@ def _cached_apply_hf_processor(
mm_missing_data_items = self._get_mm_items(mm_missing_data)
mm_missing_counts = mm_missing_data_items.get_item_counts()

# Rely on our placeholder replacement logic instead of HF
# to insert the placeholder tokens
prompt_ids = self._get_tokenizer().encode(prompt_text)

if any(count > 0 for count in mm_missing_counts.values()):
# Some HF processors (e.g. Qwen2-VL) expect corresponding
# multi-modal tokens to be in the prompt text
Expand All @@ -884,6 +880,20 @@ def _cached_apply_hf_processor(
# Avoid unnecessary tokenization of the prompt text
mm_missing_kwargs = MultiModalKwargs({})

# NOTE: Some HF processors insert BOS/EOS while others don't.
# We try to maintain consistent behavior when calling only
# the tokenizer vs when calling its parent processor
empty_ids, _ = self._apply_hf_processor(
prompt_text="",
mm_items=self._get_mm_items({}),
hf_processor_mm_kwargs={},
)
# Rely on our placeholder replacement logic instead of HF
# to insert the placeholder tokens
prompt_ids = _encode(self._get_tokenizer(),
prompt_text,
add_special_tokens=len(empty_ids) > 0)

mm_missing_next_idx = {
modality: 0
for modality in mm_missing_data_items
Expand Down

0 comments on commit bbd5997

Please sign in to comment.