Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Bugfix][Hardware][CPU] Fix CPU input_positions creation for text-only inputs with mrope #11434

Merged
merged 8 commits into from
Dec 24, 2024
+7 −9
Merged

[Bugfix][Hardware][CPU] Fix CPU input_positions creation for text-only inputs with mrope #11434

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f5a2a13
fix cpu mrope
Isotr0py Dec 23, 2024
e79a2bd
code format
Isotr0py Dec 23, 2024
9c66d68
fix vision inputs
Isotr0py Dec 23, 2024
cb5b1a3
oops
Isotr0py Dec 23, 2024
143f49f
fix mrope disable
Isotr0py Dec 23, 2024
7d6a728
fix decoding
Isotr0py Dec 23, 2024
25985eb
fix mrope compute
Isotr0py Dec 24, 2024
5a1f5f0
code format
Isotr0py Dec 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 7 additions & 9 deletions vllm/worker/cpu_model_runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,7 @@ class ModelInputData:
def __init__(self, use_mrope: bool):
self.use_mrope = use_mrope
self.input_tokens: List[int] = []
self.input_positions: Optional[
List[int]] = [] if not self.use_mrope else None
self.input_positions: List[int] = []
self.token_type_ids: Optional[List[int]] = []
self.seq_lens: List[int] = []
self.query_lens: List[int] = []
Expand All @@ -130,9 +129,8 @@ def __init__(self, use_mrope: bool):
self.multi_modal_placeholder_maps: Dict[
str, MultiModalPlaceholderMap] = defaultdict(
MultiModalPlaceholderMap)
self.input_mrope_positions: Optional[List[List[int]]] = [
[] for _ in range(3)
] if self.use_mrope else None
self.input_mrope_positions: List[List[int]] = [[]
for _ in range(3)]

def __init__(self,
runner: "CPUModelRunner",
Expand Down Expand Up @@ -167,7 +165,8 @@ def build(self) -> ModelInputForCPU:
device="cpu")
input_positions = torch.tensor(
input_data.input_positions
if not input_data.use_mrope else input_data.input_mrope_positions,
if not any(input_data.input_mrope_positions) else
input_data.input_mrope_positions,
dtype=torch.long,
device="cpu")
token_type_ids = torch.tensor(input_data.token_type_ids,
Expand Down Expand Up @@ -236,7 +235,7 @@ def _compute_decode_input_tokens(self, data: ModelInputData,
block_table = block_table[start_block:]

# For MRotaryEmbedding
if data.input_positions is None:
if seq_data.mrope_position_delta is not None:
next_pos = MRotaryEmbedding.get_next_input_positions(
seq_data.mrope_position_delta,
context_len,
Expand Down Expand Up @@ -309,8 +308,7 @@ def _compute_prompt_input_tokens(self, data: ModelInputData,
data.slot_mapping.extend(slot_mapping)

# The MROPE positions are prepared in _compute_multi_modal_input
if data.input_positions is not None:
data.input_positions.extend(token_positions)
data.input_positions.extend(token_positions)

if data.token_type_ids is not None:
data.token_type_ids.extend(token_types if token_types else [])
Expand Down
Loading