Skip to content

[Model] Rename MiniCPMVQwen2 to MiniCPMV2.6 #7273

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Aug 8, 2024
Merged

[Model] Rename MiniCPMVQwen2 to MiniCPMV2.6 #7273

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
17b8e38
Rename MiniCPMVQwen2 to MiniCPMV2.6
jeejeelee Aug 7, 2024
69fb396
fix bug
jeejeelee Aug 7, 2024
1a4bf1b
Revert resampler's kv_proj logic
jeejeelee Aug 8, 2024
25aea1c
Modify VL example
jeejeelee Aug 8, 2024
2d306b7
Merge branch 'vllm-project:main' into optimize-minicpmv-code
jeejeelee Aug 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 17 additions & 30 deletions vllm/model_executor/models/minicpmv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,16 +216,7 @@ def __init__(

self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
trunc_normal_(self.query, std=0.02)

if kv_dim is not None and kv_dim != embed_dim:
self.kv_proj = ReplicatedLinear(kv_dim, embed_dim, bias=False)
else:
# Maintain the same return value with ReplicatedLinear.forward
self.kv_proj = lambda *args, **kwargs: (
nn.Identity()(*args, **kwargs),
None,
)

self.kv_proj = ReplicatedLinear(kv_dim, embed_dim, bias=False)
self.attn = nn.MultiheadAttention(embed_dim, num_heads)
self.ln_q = norm_layer(embed_dim)
self.ln_kv = norm_layer(embed_dim)
Expand Down Expand Up @@ -261,7 +252,6 @@ def __init__(
norm_layer)

self.adaptive = adaptive

pos_embed_arr = get_2d_sincos_pos_embed(embed_dim,
grid_size,
version=(2, 0))
Expand Down Expand Up @@ -717,7 +707,7 @@ def is_default_weight_loading(self, name: str) -> bool:
raise NotImplementedError


class MiniCPMV2(MiniCPMVBaseModel):
class MiniCPMV2_0(MiniCPMVBaseModel):

def __init__(
self,
Expand Down Expand Up @@ -890,10 +880,7 @@ def is_default_weight_loading(self, name: str) -> bool:
return "resampler" in name


# NOTE: Currently, information about this model is unavailable. We are
# temporarily using `MiniCPMVQwen2` as it's name. The name may need
# to be modified in the future.
class MiniCPMVQwen2(MiniCPMVBaseModel):
class MiniCPMV2_6(MiniCPMVBaseModel):

def __init__(
self,
Expand All @@ -903,6 +890,7 @@ def __init__(
quant_config: Optional[QuantizationConfig] = None,
):
super().__init__(config, multimodal_config, cache_config, quant_config)
assert self.version == (2, 6)

def init_llm(
self,
Expand Down Expand Up @@ -930,6 +918,7 @@ def init_vision_module(self) -> nn.Module:

def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
with set_default_torch_dtype(torch.float16):
# The resampler in 2.6 remains consistent with the one in 2.5.
resampler = Resampler2_5(
num_queries=self.config.query_num,
embed_dim=embed_dim,
Expand Down Expand Up @@ -989,6 +978,13 @@ def is_default_weight_loading(self, name: str) -> bool:
return "resampler" in name or "vpm" in name


_SUPPORT_VERSION = {
(2, 0): MiniCPMV2_0,
(2, 5): MiniCPMV2_5,
(2, 6): MiniCPMV2_6
}


@MULTIMODAL_REGISTRY.register_image_input_mapper()
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_minicpmv_image_tokens)
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_minicpmv)
Expand All @@ -1007,20 +1003,11 @@ def __new__(
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
):
if not hasattr(config, "version"):
if config.hidden_size == 2304 and config.query_num == 64:
version = (2, 0)
else:
version = (2, 5)
else:
version = str(config.version).split(".")
version = tuple([int(x) for x in version])
version = str(config.version).split(".")
version = tuple([int(x) for x in version])
# Dispatch class based on version
if version == (2, 0):
instance_class = MiniCPMV2
elif version == (2, 5):
instance_class = MiniCPMV2_5
else:
instance_class = MiniCPMVQwen2
instance_class = _SUPPORT_VERSION.get(version, None)
if instance_class is None:
raise NotImplementedError
return instance_class(config, multimodal_config, cache_config,
quant_config)
Loading