Try to fix Phi3V and Ultravox

Signed-off-by: DarkLight1337 <[email protected]>

tests/multimodal/test_processing.py

@@ -618,7 +618,8 @@ def _test_processing_cache_correctness(
             hf_processor_mm_kwargs={},
         )
 
-        assert baseline_result == cached_result, f"Failed (idx={batch_idx})"
+        assert baseline_result == cached_result, (
+            f"Failed ({batch_idx=}, {mm_data=})")
 
 
 # yapf: disable

vllm/multimodal/processing.py

@@ -866,10 +866,6 @@ def _cached_apply_hf_processor(
         mm_missing_data_items = self._get_mm_items(mm_missing_data)
         mm_missing_counts = mm_missing_data_items.get_item_counts()
 
-        # Rely on our placeholder replacement logic instead of HF
-        # to insert the placeholder tokens
-        prompt_ids = self._get_tokenizer().encode(prompt_text)
-
         if any(count > 0 for count in mm_missing_counts.values()):
             # Some HF processors (e.g. Qwen2-VL) expect corresponding
             # multi-modal tokens to be in the prompt text
@@ -884,6 +880,20 @@ def _cached_apply_hf_processor(
             # Avoid unnecessary tokenization of the prompt text
             mm_missing_kwargs = MultiModalKwargs({})
 
+        # NOTE: Some HF processors insert BOS/EOS while others don't.
+        # We try to maintain consistent behavior when calling only
+        # the tokenizer vs when calling its parent processor
+        empty_ids, _ = self._apply_hf_processor(
+            prompt_text="",
+            mm_items=self._get_mm_items({}),
+            hf_processor_mm_kwargs={},
+        )
+        # Rely on our placeholder replacement logic instead of HF
+        # to insert the placeholder tokens
+        prompt_ids = _encode(self._get_tokenizer(),
+                             prompt_text,
+                             add_special_tokens=len(empty_ids) > 0)
+
         mm_missing_next_idx = {
             modality: 0
             for modality in mm_missing_data_items