From 53a4ed7e6e162619af32d95aa9f4e6d32414214d Mon Sep 17 00:00:00 2001 From: ZTWHHH Date: Wed, 10 Jun 2026 15:23:24 +0000 Subject: [PATCH 1/6] Add 5 new model series: kimi_vl, minicpm_v_4d5, glm_4d1v, ovis2, aria - mmeval/infer/{kimi_vl,minicpm_v_4d5,glm_4d1v,ovis2,aria}.py - env_files/{kimi_vl,minicpm_v_4d5,glm_4d1v,ovis2,aria}_requirements.txt - mmeval/registry.py: series_mapping + series_infer_env_mapping entries Models: - kimi_vl: moonshotai/Kimi-VL-A3B-Instruct, Kimi-VL-A3B-Thinking - minicpm_v_4d5: openbmb/MiniCPM-V-4_5 - glm_4d1v: THUDM/GLM-4.1V-9B-Thinking, GLM-4.1V-9B-Base - ovis2: AIDC-AI/Ovis2-{1B,2B,4B,8B,16B,34B} - aria: rhymes-ai/Aria Co-Authored-By: Claude Opus 4.7 --- env_files/aria_requirements.txt | 12 +++ env_files/glm_4d1v_requirements.txt | 12 +++ env_files/kimi_vl_requirements.txt | 14 +++ env_files/minicpm_v_4d5_requirements.txt | 13 +++ env_files/ovis2_requirements.txt | 12 +++ mmeval/infer/aria.py | 106 +++++++++++++++++++++ mmeval/infer/glm_4d1v.py | 96 +++++++++++++++++++ mmeval/infer/kimi_vl.py | 110 +++++++++++++++++++++ mmeval/infer/minicpm_v_4d5.py | 93 ++++++++++++++++++ mmeval/infer/ovis2.py | 116 +++++++++++++++++++++++ mmeval/registry.py | 25 +++++ 11 files changed, 609 insertions(+) create mode 100644 env_files/aria_requirements.txt create mode 100644 env_files/glm_4d1v_requirements.txt create mode 100644 env_files/kimi_vl_requirements.txt create mode 100644 env_files/minicpm_v_4d5_requirements.txt create mode 100644 env_files/ovis2_requirements.txt create mode 100644 mmeval/infer/aria.py create mode 100644 mmeval/infer/glm_4d1v.py create mode 100644 mmeval/infer/kimi_vl.py create mode 100644 mmeval/infer/minicpm_v_4d5.py create mode 100644 mmeval/infer/ovis2.py diff --git a/env_files/aria_requirements.txt b/env_files/aria_requirements.txt new file mode 100644 index 00000000..3e069fc1 --- /dev/null +++ b/env_files/aria_requirements.txt @@ -0,0 +1,12 @@ +torch==2.4.0 +torchvision==0.19.0 +transformers==4.49.0 +accelerate +datasets +pandas +numpy==1.26.4 +Pillow +sentencepiece +protobuf +einops +timm diff --git a/env_files/glm_4d1v_requirements.txt b/env_files/glm_4d1v_requirements.txt new file mode 100644 index 00000000..6004f8af --- /dev/null +++ b/env_files/glm_4d1v_requirements.txt @@ -0,0 +1,12 @@ +torch==2.6.0 +torchvision==0.21.0 +transformers==4.57.1 +accelerate +datasets +pandas +numpy==1.26.4 +Pillow +sentencepiece +protobuf +einops +timm diff --git a/env_files/kimi_vl_requirements.txt b/env_files/kimi_vl_requirements.txt new file mode 100644 index 00000000..2d02a3eb --- /dev/null +++ b/env_files/kimi_vl_requirements.txt @@ -0,0 +1,14 @@ +torch==2.4.0 +torchvision==0.19.0 +transformers==4.48.2 +accelerate +datasets +pandas +numpy==1.26.4 +Pillow +sentencepiece +protobuf +einops +timm +tiktoken +blobfile diff --git a/env_files/minicpm_v_4d5_requirements.txt b/env_files/minicpm_v_4d5_requirements.txt new file mode 100644 index 00000000..f2935321 --- /dev/null +++ b/env_files/minicpm_v_4d5_requirements.txt @@ -0,0 +1,13 @@ +torch==2.4.0 +torchvision==0.19.0 +transformers==4.55.0 +accelerate +datasets +pandas +numpy==1.26.4 +Pillow +sentencepiece +protobuf +einops +timm +decord diff --git a/env_files/ovis2_requirements.txt b/env_files/ovis2_requirements.txt new file mode 100644 index 00000000..946e5b29 --- /dev/null +++ b/env_files/ovis2_requirements.txt @@ -0,0 +1,12 @@ +torch==2.4.0 +torchvision==0.19.0 +transformers==4.46.2 +accelerate +datasets +pandas +numpy==1.26.4 +Pillow +sentencepiece +protobuf +einops +timm diff --git a/mmeval/infer/aria.py b/mmeval/infer/aria.py new file mode 100644 index 00000000..f3d36f89 --- /dev/null +++ b/mmeval/infer/aria.py @@ -0,0 +1,106 @@ +"""aria — rhymes-ai/Aria. + +HF: https://huggingface.co/rhymes-ai/Aria +GH: https://github.com/rhymes-ai/Aria +Paper: https://arxiv.org/abs/2410.05993 +""" +import re +import copy + +import torch +from PIL import Image +from transformers import AriaForConditionalGeneration, AriaProcessor + +from mmeval.infer.task import Task +from mmeval.utils import constants +from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs + + +class TaskRunner(Task): + def __init__(self, args): + self.args = args + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.dtype = getattr(args, "dtype") or torch.bfloat16 + self.default_model_kwargs = {"device_map": "auto"} + self.default_gen_kwargs = { + "max_new_tokens": 512, + "do_sample": True, + "temperature": 0.9, + } + self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs) + self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs) + + super().__init__(args) + + def load_model(self, args): + self.model = AriaForConditionalGeneration.from_pretrained( + args.model_name_or_path, + torch_dtype=self.dtype, + **self.model_kwargs, + ).eval() + self.processor = AriaProcessor.from_pretrained(args.model_name_or_path) + + def parse_input(self, message): + question = message["prompt"] + q_chunks = re.split(r'(<(?:image|video)>)', question) + media_list = message.get('media', []) + + content = [] + images = [] + media_idx = 0 + for chunk in q_chunks: + if not chunk.strip(): + continue + if chunk == constants.image: + img = media_list[media_idx] + if isinstance(img, str): + img = Image.open(img).convert("RGB") + elif hasattr(img, "convert"): + img = img.convert("RGB") + images.append(img) + content.append({"type": "image"}) + media_idx += 1 + elif chunk == constants.video: + raise NotImplementedError("aria video input not implemented") + else: + content.append({"type": "text", "text": chunk}) + return [{"role": "user", "content": content}], images + + def _generate_response(self, inputs): + output = self.model.generate( + **inputs, + stop_strings=["<|im_end|>"], + tokenizer=self.processor.tokenizer, + **self.gen_kwargs, + ) + trimmed = output[0][inputs["input_ids"].shape[1]:] + return self.processor.decode(trimmed, skip_special_tokens=True) + + def run_sample(self, sample: dict): + if self.args.score_target: + raise NotImplementedError( + "aria: score_target is not implemented yet" + ) + + ori_sample = copy.deepcopy(sample) + message = sample["messages"][0] + messages, images = self.parse_input(message) + + text = self.processor.apply_chat_template(messages, add_generation_prompt=True) + proc_kwargs = {"text": text, "return_tensors": "pt"} + if images: + proc_kwargs["images"] = images + inputs = self.processor(**proc_kwargs) + if "pixel_values" in inputs: + inputs["pixel_values"] = inputs["pixel_values"].to(self.dtype) + inputs = inputs.to(self.model.device) + + response = self._generate_response(inputs) + ori_sample["messages"].append({"role": "assistant", "response": response}) + return ori_sample + + +if __name__ == "__main__": + args = parse_args() + model_evaluator = TaskRunner(args) + model_evaluator.inference_dataset() diff --git a/mmeval/infer/glm_4d1v.py b/mmeval/infer/glm_4d1v.py new file mode 100644 index 00000000..6f666adf --- /dev/null +++ b/mmeval/infer/glm_4d1v.py @@ -0,0 +1,96 @@ +"""glm_4d1v — GLM-4.1V-9B-Thinking. + +HF: https://huggingface.co/THUDM/GLM-4.1V-9B-Thinking +GH: https://github.com/THUDM/GLM-V +""" +import re +import copy + +import torch +from PIL import Image +from transformers import AutoProcessor, Glm4vForConditionalGeneration + +from mmeval.infer.task import Task +from mmeval.utils import constants +from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs + + +class TaskRunner(Task): + def __init__(self, args): + self.args = args + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.dtype = getattr(args, "dtype") or torch.bfloat16 + self.default_model_kwargs = {"device_map": "auto"} + self.default_gen_kwargs = {"max_new_tokens": 8192} + self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs) + self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs) + + super().__init__(args) + + def load_model(self, args): + self.model = Glm4vForConditionalGeneration.from_pretrained( + args.model_name_or_path, + torch_dtype=self.dtype, + **self.model_kwargs, + ).eval() + self.processor = AutoProcessor.from_pretrained( + args.model_name_or_path, use_fast=True, + ) + + def parse_input(self, message): + question = message["prompt"] + q_chunks = re.split(r'(<(?:image|video)>)', question) + media_list = message.get('media', []) + + content = [] + media_idx = 0 + for chunk in q_chunks: + if not chunk.strip(): + continue + if chunk == constants.image: + img = media_list[media_idx] + if isinstance(img, str): + img = Image.open(img).convert("RGB") + elif hasattr(img, "convert"): + img = img.convert("RGB") + content.append({"type": "image", "image": img}) + media_idx += 1 + elif chunk == constants.video: + content.append({"type": "video", "video": media_list[media_idx]}) + media_idx += 1 + else: + content.append({"type": "text", "text": chunk}) + return [{"role": "user", "content": content}] + + def _generate_response(self, inputs): + generated_ids = self.model.generate(**inputs, **self.gen_kwargs) + trimmed = generated_ids[0][inputs["input_ids"].shape[1]:] + return self.processor.decode(trimmed, skip_special_tokens=False) + + def run_sample(self, sample: dict): + if self.args.score_target: + raise NotImplementedError( + "glm_4d1v: score_target is not implemented yet" + ) + + ori_sample = copy.deepcopy(sample) + message = sample["messages"][0] + messages = self.parse_input(message) + + inputs = self.processor.apply_chat_template( + messages, + tokenize=True, + add_generation_prompt=True, + return_dict=True, + return_tensors="pt", + ).to(self.model.device) + + response = self._generate_response(inputs) + ori_sample["messages"].append({"role": "assistant", "response": response}) + return ori_sample + + +if __name__ == "__main__": + args = parse_args() + model_evaluator = TaskRunner(args) + model_evaluator.inference_dataset() diff --git a/mmeval/infer/kimi_vl.py b/mmeval/infer/kimi_vl.py new file mode 100644 index 00000000..f39cd2c8 --- /dev/null +++ b/mmeval/infer/kimi_vl.py @@ -0,0 +1,110 @@ +"""kimi_vl — Kimi-VL-A3B-{Instruct,Thinking}. + +HF: https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct +GH: https://github.com/MoonshotAI/Kimi-VL +Paper: https://arxiv.org/abs/2504.07491 +""" +import re +import copy + +import torch +from PIL import Image +from transformers import AutoModelForCausalLM, AutoProcessor + +from mmeval.infer.task import Task +from mmeval.utils import constants +from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs + + +class TaskRunner(Task): + def __init__(self, args): + self.args = args + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.dtype = getattr(args, "dtype") or "auto" + self.default_model_kwargs = {"device_map": "auto"} + self.default_gen_kwargs = {"max_new_tokens": 512} + self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs) + self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs) + + super().__init__(args) + + def load_model(self, args): + self.model = AutoModelForCausalLM.from_pretrained( + args.model_name_or_path, + torch_dtype=self.dtype, + trust_remote_code=True, + **self.model_kwargs, + ).eval() + self.processor = AutoProcessor.from_pretrained( + args.model_name_or_path, trust_remote_code=True, + ) + + def parse_input(self, message): + question = message["prompt"] + q_chunks = re.split(r'(<(?:image|video)>)', question) + media_list = message.get('media', []) + + content = [] + media_idx = 0 + for chunk in q_chunks: + if not chunk.strip(): + continue + if chunk == constants.image: + content.append({"type": "image", "image": media_list[media_idx]}) + media_idx += 1 + elif chunk == constants.video: + raise NotImplementedError("kimi_vl video input not implemented") + else: + content.append({"type": "text", "text": chunk}) + return [{"role": "user", "content": content}] + + def _materialize_images(self, user_message): + images = [] + for turn in user_message: + for item in turn["content"]: + if item.get("type") == "image": + img = item["image"] + if isinstance(img, str): + img = Image.open(img).convert("RGB") + elif hasattr(img, "convert"): + img = img.convert("RGB") + images.append(img) + return images + + def _generate_response(self, inputs): + generated_ids = self.model.generate(**inputs, **self.gen_kwargs) + trimmed = [ + out[len(inp):] for inp, out in zip(inputs.input_ids, generated_ids) + ] + return self.processor.batch_decode( + trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False, + )[0] + + def run_sample(self, sample: dict): + if self.args.score_target: + raise NotImplementedError( + "kimi_vl: score_target is not implemented yet" + ) + + ori_sample = copy.deepcopy(sample) + message = sample["messages"][0] + user_message = self.parse_input(message) + images = self._materialize_images(user_message) + + text = self.processor.apply_chat_template( + user_message, add_generation_prompt=True, return_tensors="pt", + ) + proc_kwargs = {"text": text, "return_tensors": "pt", "padding": True, "truncation": True} + if images: + proc_kwargs["images"] = images + inputs = self.processor(**proc_kwargs).to(self.model.device) + + response = self._generate_response(inputs) + ori_sample["messages"].append({"role": "assistant", "response": response}) + return ori_sample + + +if __name__ == "__main__": + args = parse_args() + model_evaluator = TaskRunner(args) + model_evaluator.inference_dataset() diff --git a/mmeval/infer/minicpm_v_4d5.py b/mmeval/infer/minicpm_v_4d5.py new file mode 100644 index 00000000..7b5b8e8d --- /dev/null +++ b/mmeval/infer/minicpm_v_4d5.py @@ -0,0 +1,93 @@ +"""minicpm_v_4d5 — MiniCPM-V-4_5. + +HF: https://huggingface.co/openbmb/MiniCPM-V-4_5 +GH: https://github.com/OpenBMB/MiniCPM-o +""" +import re +import copy + +import torch +from PIL import Image +from transformers import AutoModel, AutoTokenizer + +from mmeval.infer.task import Task +from mmeval.utils import constants +from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs + + +class TaskRunner(Task): + def __init__(self, args): + self.args = args + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.dtype = getattr(args, "dtype") or torch.bfloat16 + self.default_model_kwargs = { + "attn_implementation": "sdpa", + } + self.default_gen_kwargs = {"max_new_tokens": 1024} + self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs) + self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs) + + super().__init__(args) + + def load_model(self, args): + self.model = AutoModel.from_pretrained( + args.model_name_or_path, + torch_dtype=self.dtype, + trust_remote_code=True, + **self.model_kwargs, + ).eval().cuda() + self.tokenizer = AutoTokenizer.from_pretrained( + args.model_name_or_path, trust_remote_code=True, + ) + + def parse_input(self, message): + question = message["prompt"] + q_chunks = re.split(r'(<(?:image|video)>)', question) + media_list = message.get('media', []) + + content = [] + text_parts = [] + media_idx = 0 + for chunk in q_chunks: + if not chunk.strip(): + continue + if chunk == constants.image: + img = media_list[media_idx] + if isinstance(img, str): + img = Image.open(img).convert("RGB") + elif hasattr(img, "convert"): + img = img.convert("RGB") + content.append(img) + media_idx += 1 + elif chunk == constants.video: + raise NotImplementedError("minicpm_v_4d5 video input not implemented") + else: + text_parts.append(chunk) + content.append("".join(text_parts).strip()) + return [{"role": "user", "content": content}] + + def run_sample(self, sample: dict): + if self.args.score_target: + raise NotImplementedError( + "minicpm_v_4d5: score_target is not implemented yet" + ) + + ori_sample = copy.deepcopy(sample) + message = sample["messages"][0] + msgs = self.parse_input(message) + + response = self.model.chat( + msgs=msgs, + tokenizer=self.tokenizer, + enable_thinking=False, + stream=False, + **self.gen_kwargs, + ) + ori_sample["messages"].append({"role": "assistant", "response": response}) + return ori_sample + + +if __name__ == "__main__": + args = parse_args() + model_evaluator = TaskRunner(args) + model_evaluator.inference_dataset() diff --git a/mmeval/infer/ovis2.py b/mmeval/infer/ovis2.py new file mode 100644 index 00000000..fa216d0c --- /dev/null +++ b/mmeval/infer/ovis2.py @@ -0,0 +1,116 @@ +"""ovis2 — AIDC-AI/Ovis2-{1B,2B,4B,8B,16B,34B}. + +HF: https://huggingface.co/AIDC-AI/Ovis2-8B +GH: https://github.com/AIDC-AI/Ovis +Paper: https://arxiv.org/abs/2405.20797 +""" +import re +import copy + +import torch +from PIL import Image +from transformers import AutoModelForCausalLM + +from mmeval.infer.task import Task +from mmeval.utils import constants +from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs + + +class TaskRunner(Task): + def __init__(self, args): + self.args = args + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.dtype = getattr(args, "dtype") or torch.bfloat16 + self.default_model_kwargs = {} + self.default_gen_kwargs = { + "max_new_tokens": 1024, + "do_sample": False, + } + self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs) + self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs) + + super().__init__(args) + + def load_model(self, args): + self.model = AutoModelForCausalLM.from_pretrained( + args.model_name_or_path, + torch_dtype=self.dtype, + trust_remote_code=True, + multimodal_max_length=32768, + **self.model_kwargs, + ).cuda().eval() + self.text_tokenizer = self.model.get_text_tokenizer() + self.visual_tokenizer = self.model.get_visual_tokenizer() + self.max_partition = 9 + + def parse_input(self, message): + question = message["prompt"] + q_chunks = re.split(r'(<(?:image|video)>)', question) + media_list = message.get('media', []) + + text_parts = [] + images = [] + media_idx = 0 + for chunk in q_chunks: + if not chunk.strip(): + continue + if chunk == constants.image: + img = media_list[media_idx] + if isinstance(img, str): + img = Image.open(img).convert("RGB") + elif hasattr(img, "convert"): + img = img.convert("RGB") + images.append(img) + text_parts.append("") + media_idx += 1 + elif chunk == constants.video: + raise NotImplementedError("ovis2 video input not implemented") + else: + text_parts.append(chunk) + return "".join(text_parts).strip(), images + + def run_sample(self, sample: dict): + if self.args.score_target: + raise NotImplementedError( + "ovis2: score_target is not implemented yet" + ) + + ori_sample = copy.deepcopy(sample) + message = sample["messages"][0] + query, images = self.parse_input(message) + if images and "" not in query: + query = "\n" + query + + prompt, input_ids, pixel_values = self.model.preprocess_inputs( + query, images if images else None, max_partition=self.max_partition, + ) + attention_mask = torch.ne(input_ids, self.text_tokenizer.pad_token_id) + input_ids = input_ids.unsqueeze(0).to(self.model.device) + attention_mask = attention_mask.unsqueeze(0).to(self.model.device) + if pixel_values is not None: + pixel_values = [ + pixel_values.to(dtype=self.visual_tokenizer.dtype, + device=self.visual_tokenizer.device) + ] + else: + pixel_values = [None] + + with torch.inference_mode(): + output_ids = self.model.generate( + input_ids, + pixel_values=pixel_values, + attention_mask=attention_mask, + **self.gen_kwargs, + eos_token_id=self.model.generation_config.eos_token_id, + pad_token_id=self.text_tokenizer.pad_token_id, + use_cache=True, + )[0] + response = self.text_tokenizer.decode(output_ids, skip_special_tokens=True) + ori_sample["messages"].append({"role": "assistant", "response": response}) + return ori_sample + + +if __name__ == "__main__": + args = parse_args() + model_evaluator = TaskRunner(args) + model_evaluator.inference_dataset() diff --git a/mmeval/registry.py b/mmeval/registry.py index dfbf0b98..a7deb7d4 100644 --- a/mmeval/registry.py +++ b/mmeval/registry.py @@ -77,6 +77,11 @@ "doubao-seed-2-0-mini-260215", "doubao-seed-2-0-lite-260215", "doubao-seed-2-0-code-preview-260215", "doubao-seed-2-0-pro-260215"], "hunyuan_vision": ["hunyuan-vision", "hunyuan-vision-1.5-instruct", "hunyuan-t1-vision", "hunyuan-turbos-vision", "hunyuan-large-vision"], "cosmos_reason2": ["Cosmos-Reason2-2B", "Cosmos-Reason2-8B"], + "kimi_vl": ["Kimi-VL-A3B-Instruct", "Kimi-VL-A3B-Thinking"], + "minicpm_v_4d5": ["MiniCPM-V-4_5"], + "glm_4d1v": ["GLM-4.1V-9B-Thinking", "GLM-4.1V-9B-Base"], + "ovis2": ["Ovis2-1B", "Ovis2-2B", "Ovis2-4B", "Ovis2-8B", "Ovis2-16B", "Ovis2-34B"], + "aria": ["Aria"], } series_infer_env_mapping = { @@ -316,4 +321,24 @@ "env": os.path.join(env_dir, "cosmos_reason2"), "infer_file": "cosmos_reason2.py", }, + "kimi_vl": { + "env": os.path.join(env_dir, "kimi_vl"), + "infer_file": "kimi_vl.py", + }, + "minicpm_v_4d5": { + "env": os.path.join(env_dir, "minicpm_v_4d5"), + "infer_file": "minicpm_v_4d5.py", + }, + "glm_4d1v": { + "env": os.path.join(env_dir, "glm_4d1v"), + "infer_file": "glm_4d1v.py", + }, + "ovis2": { + "env": os.path.join(env_dir, "ovis2"), + "infer_file": "ovis2.py", + }, + "aria": { + "env": os.path.join(env_dir, "aria"), + "infer_file": "aria.py", + }, } From 68d7c337e57c118ec4ef28ec9f8b4a34316ff31a Mon Sep 17 00:00:00 2001 From: ZTWHHH Date: Wed, 10 Jun 2026 16:01:31 +0000 Subject: [PATCH 2/6] Add qwen3d5 (Qwen3.5 unified vision-language) integration Adds the official Qwen3.5 multimodal series to the same branch alongside the previous 5 series. Qwen3.5 is a unified vision-language model (no separate -VL suffix); architecture: Qwen3_5ForConditionalGeneration with Qwen3VLProcessor. Models: Qwen/Qwen3.5-{0.8B,2B,4B,9B,27B,35B-A3B,122B-A10B,397B-A17B} and Base / FP8 / GPTQ-Int4 variants. HF: https://huggingface.co/Qwen/Qwen3.5-9B GH: https://github.com/QwenLM/Qwen3.5 Co-Authored-By: Claude Opus 4.7 --- env_files/qwen3d5_requirements.txt | 14 ++++ mmeval/infer/qwen3d5.py | 128 +++++++++++++++++++++++++++++ mmeval/registry.py | 5 ++ 3 files changed, 147 insertions(+) create mode 100644 env_files/qwen3d5_requirements.txt create mode 100644 mmeval/infer/qwen3d5.py diff --git a/env_files/qwen3d5_requirements.txt b/env_files/qwen3d5_requirements.txt new file mode 100644 index 00000000..72138086 --- /dev/null +++ b/env_files/qwen3d5_requirements.txt @@ -0,0 +1,14 @@ +torch==2.6.0 +torchvision==0.21.0 +transformers @ git+https://github.com/huggingface/transformers.git@main +accelerate +datasets +pandas +numpy==1.26.4 +Pillow +sentencepiece +protobuf +einops +timm +qwen_vl_utils +av diff --git a/mmeval/infer/qwen3d5.py b/mmeval/infer/qwen3d5.py new file mode 100644 index 00000000..97ca9bac --- /dev/null +++ b/mmeval/infer/qwen3d5.py @@ -0,0 +1,128 @@ +"""qwen3d5 — Qwen3.5 unified vision-language family. + +HF: https://huggingface.co/Qwen/Qwen3.5-9B +GH: https://github.com/QwenLM/Qwen3.5 +Blog: https://qwen.ai/blog?id=qwen3.5 +""" +import re +import copy + +import torch +from transformers import AutoModelForImageTextToText, AutoProcessor +from qwen_vl_utils import process_vision_info + +from mmeval.infer.task import Task +from mmeval.utils import constants +from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs + + +class TaskRunner(Task): + def __init__(self, args): + self.args = args + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.dtype = getattr(args, "dtype") or "auto" + self.default_model_kwargs = {"device_map": "auto"} + self.default_gen_kwargs = {"max_new_tokens": 512} + self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs) + self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs) + + super().__init__(args) + + def load_model(self, args): + self.model = AutoModelForImageTextToText.from_pretrained( + args.model_name_or_path, + dtype=self.dtype, + **self.model_kwargs, + ) + self.processor = AutoProcessor.from_pretrained(args.model_name_or_path) + + def parse_input(self, message): + question = message["prompt"] + q_chunks = re.split(r'(<(?:image|video)>)', question) + media_list = message.get('media', []) + + messages = [{"role": "user", "content": []}] + media_idx = 0 + for chunk in q_chunks: + if not chunk.strip(): + continue + if chunk == constants.image: + media = media_list[media_idx] + media_idx += 1 + messages[0]["content"].append({ + "type": "image", + "image": media, + "min_pixels": 4 * 32 * 32, + "max_pixels": 256 * 32 * 32, + }) + elif chunk == constants.video: + media = media_list[media_idx] + media_idx += 1 + messages[0]["content"].append({ + "type": "video", + "video": media, + "min_pixels": 4 * 32 * 32, + "max_pixels": 256 * 32 * 32, + "total_pixels": 20480 * 32 * 32, + }) + else: + messages[0]["content"].append({"type": "text", "text": chunk}) + return messages + + def _generate_response(self, inputs): + generated_ids = self.model.generate(**inputs, **self.gen_kwargs) + generated_ids_trimmed = [ + out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids) + ] + return self.processor.batch_decode( + generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False, + ) + + def run_sample(self, sample: dict): + if self.args.score_target: + raise NotImplementedError( + "qwen3d5: score_target is not implemented yet" + ) + + ori_sample = copy.deepcopy(sample) + message = sample["messages"][0] + + user_message = self.parse_input(message) + + text = self.processor.apply_chat_template( + user_message, tokenize=False, add_generation_prompt=True + ) + + images, videos, video_kwargs = process_vision_info( + user_message, + image_patch_size=16, + return_video_kwargs=True, + return_video_metadata=True, + ) + + if videos is not None: + videos, video_metadatas = zip(*videos) + videos, video_metadatas = list(videos), list(video_metadatas) + else: + video_metadatas = None + + inputs = self.processor( + text=text, + images=images, + videos=videos, + video_metadata=video_metadatas, + return_tensors="pt", + do_resize=False, + **video_kwargs, + ) + inputs = inputs.to(self.model.device) + + response = self._generate_response(inputs) + ori_sample["messages"].append({"role": "assistant", "response": response}) + return ori_sample + + +if __name__ == "__main__": + args = parse_args() + model_evaluator = TaskRunner(args) + model_evaluator.inference_dataset() diff --git a/mmeval/registry.py b/mmeval/registry.py index a7deb7d4..428de0bd 100644 --- a/mmeval/registry.py +++ b/mmeval/registry.py @@ -82,6 +82,7 @@ "glm_4d1v": ["GLM-4.1V-9B-Thinking", "GLM-4.1V-9B-Base"], "ovis2": ["Ovis2-1B", "Ovis2-2B", "Ovis2-4B", "Ovis2-8B", "Ovis2-16B", "Ovis2-34B"], "aria": ["Aria"], + "qwen3d5": ["Qwen3.5-0.8B", "Qwen3.5-2B", "Qwen3.5-4B", "Qwen3.5-9B", "Qwen3.5-27B", "Qwen3.5-35B-A3B", "Qwen3.5-122B-A10B", "Qwen3.5-397B-A17B", "Qwen3.5-0.8B-Base", "Qwen3.5-4B-Base", "Qwen3.5-9B-Base", "Qwen3.5-35B-A3B-Base", "Qwen3.5-27B-FP8", "Qwen3.5-122B-A10B-FP8", "Qwen3.5-397B-A17B-FP8", "Qwen3.5-35B-A3B-GPTQ-Int4", "Qwen3.5-122B-A10B-GPTQ-Int4"], } series_infer_env_mapping = { @@ -341,4 +342,8 @@ "env": os.path.join(env_dir, "aria"), "infer_file": "aria.py", }, + "qwen3d5": { + "env": os.path.join(env_dir, "qwen3d5"), + "infer_file": "qwen3d5.py", + }, } From 9a99fafc11e0d4e29ff14cc9749f0fcee67a79b7 Mon Sep 17 00:00:00 2001 From: ZTWHHH Date: Thu, 11 Jun 2026 07:51:38 +0000 Subject: [PATCH 3/6] Restrict ovis2/qwen3d5 to tested variants; add av to glm_4d1v reqs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ovis2: registered variants narrowed to Ovis2-{1B,2B,4B,8B} (all tested). Larger Ovis2-{16B,34B} variants are de-registered to keep the registry to tested-only variants. - qwen3d5: registered variants narrowed to Qwen3.5-{0.8B,2B,4B,9B}. Larger variants (27B / 35B-A3B / 122B-A10B / 397B-A17B and their Base/FP8/GPTQ flavors) are de-registered for the same reason; they can be reintroduced in a follow-up once tested. - env_files/glm_4d1v_requirements.txt: add av (PyAV) — the Glm4v processor's video path requires it via torchvision.io. --- env_files/glm_4d1v_requirements.txt | 1 + mmeval/registry.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/env_files/glm_4d1v_requirements.txt b/env_files/glm_4d1v_requirements.txt index 6004f8af..0ef1cff7 100644 --- a/env_files/glm_4d1v_requirements.txt +++ b/env_files/glm_4d1v_requirements.txt @@ -10,3 +10,4 @@ sentencepiece protobuf einops timm +av diff --git a/mmeval/registry.py b/mmeval/registry.py index 428de0bd..5a18dc0f 100644 --- a/mmeval/registry.py +++ b/mmeval/registry.py @@ -80,9 +80,9 @@ "kimi_vl": ["Kimi-VL-A3B-Instruct", "Kimi-VL-A3B-Thinking"], "minicpm_v_4d5": ["MiniCPM-V-4_5"], "glm_4d1v": ["GLM-4.1V-9B-Thinking", "GLM-4.1V-9B-Base"], - "ovis2": ["Ovis2-1B", "Ovis2-2B", "Ovis2-4B", "Ovis2-8B", "Ovis2-16B", "Ovis2-34B"], + "ovis2": ["Ovis2-1B", "Ovis2-2B", "Ovis2-4B", "Ovis2-8B"], "aria": ["Aria"], - "qwen3d5": ["Qwen3.5-0.8B", "Qwen3.5-2B", "Qwen3.5-4B", "Qwen3.5-9B", "Qwen3.5-27B", "Qwen3.5-35B-A3B", "Qwen3.5-122B-A10B", "Qwen3.5-397B-A17B", "Qwen3.5-0.8B-Base", "Qwen3.5-4B-Base", "Qwen3.5-9B-Base", "Qwen3.5-35B-A3B-Base", "Qwen3.5-27B-FP8", "Qwen3.5-122B-A10B-FP8", "Qwen3.5-397B-A17B-FP8", "Qwen3.5-35B-A3B-GPTQ-Int4", "Qwen3.5-122B-A10B-GPTQ-Int4"], + "qwen3d5": ["Qwen3.5-0.8B", "Qwen3.5-2B", "Qwen3.5-4B", "Qwen3.5-9B"], } series_infer_env_mapping = { From b161f74bb3f648577f2b94edf4d9fe4574000c59 Mon Sep 17 00:00:00 2001 From: ZTWHHH Date: Thu, 11 Jun 2026 08:10:37 +0000 Subject: [PATCH 4/6] minicpm_v_4d5: preserve interleave order in parse_input The previous parse_input concatenated all text fragments and appended them at the end of the content list, breaking interleave order for prompts where images appear between text chunks (e.g. 'Image A and Image B '). Append text chunks in position so the multi-image interleave path matches the official model.chat() expectations. --- mmeval/infer/minicpm_v_4d5.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mmeval/infer/minicpm_v_4d5.py b/mmeval/infer/minicpm_v_4d5.py index 7b5b8e8d..8ecbc583 100644 --- a/mmeval/infer/minicpm_v_4d5.py +++ b/mmeval/infer/minicpm_v_4d5.py @@ -46,7 +46,6 @@ def parse_input(self, message): media_list = message.get('media', []) content = [] - text_parts = [] media_idx = 0 for chunk in q_chunks: if not chunk.strip(): @@ -62,8 +61,7 @@ def parse_input(self, message): elif chunk == constants.video: raise NotImplementedError("minicpm_v_4d5 video input not implemented") else: - text_parts.append(chunk) - content.append("".join(text_parts).strip()) + content.append(chunk) return [{"role": "user", "content": content}] def run_sample(self, sample: dict): From f115d8834f2af193ae6f611851aa57e42f63fe25 Mon Sep 17 00:00:00 2001 From: ZTWHHH Date: Thu, 11 Jun 2026 09:08:06 +0000 Subject: [PATCH 5/6] Raise default max_new_tokens to 2048 for the 6 new inference files Previous defaults (512/1024/8192) replaced with a uniform 2048 for the six newly added series. 2048 is the project-wide normal upper bound; no model needs a lower limit. --- mmeval/infer/aria.py | 2 +- mmeval/infer/glm_4d1v.py | 2 +- mmeval/infer/kimi_vl.py | 2 +- mmeval/infer/minicpm_v_4d5.py | 2 +- mmeval/infer/ovis2.py | 2 +- mmeval/infer/qwen3d5.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mmeval/infer/aria.py b/mmeval/infer/aria.py index f3d36f89..1b1c24e2 100644 --- a/mmeval/infer/aria.py +++ b/mmeval/infer/aria.py @@ -23,7 +23,7 @@ def __init__(self, args): self.dtype = getattr(args, "dtype") or torch.bfloat16 self.default_model_kwargs = {"device_map": "auto"} self.default_gen_kwargs = { - "max_new_tokens": 512, + "max_new_tokens": 2048, "do_sample": True, "temperature": 0.9, } diff --git a/mmeval/infer/glm_4d1v.py b/mmeval/infer/glm_4d1v.py index 6f666adf..6e3534d5 100644 --- a/mmeval/infer/glm_4d1v.py +++ b/mmeval/infer/glm_4d1v.py @@ -21,7 +21,7 @@ def __init__(self, args): self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.dtype = getattr(args, "dtype") or torch.bfloat16 self.default_model_kwargs = {"device_map": "auto"} - self.default_gen_kwargs = {"max_new_tokens": 8192} + self.default_gen_kwargs = {"max_new_tokens": 2048} self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs) self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs) diff --git a/mmeval/infer/kimi_vl.py b/mmeval/infer/kimi_vl.py index f39cd2c8..4e80036d 100644 --- a/mmeval/infer/kimi_vl.py +++ b/mmeval/infer/kimi_vl.py @@ -22,7 +22,7 @@ def __init__(self, args): self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.dtype = getattr(args, "dtype") or "auto" self.default_model_kwargs = {"device_map": "auto"} - self.default_gen_kwargs = {"max_new_tokens": 512} + self.default_gen_kwargs = {"max_new_tokens": 2048} self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs) self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs) diff --git a/mmeval/infer/minicpm_v_4d5.py b/mmeval/infer/minicpm_v_4d5.py index 8ecbc583..733abac6 100644 --- a/mmeval/infer/minicpm_v_4d5.py +++ b/mmeval/infer/minicpm_v_4d5.py @@ -23,7 +23,7 @@ def __init__(self, args): self.default_model_kwargs = { "attn_implementation": "sdpa", } - self.default_gen_kwargs = {"max_new_tokens": 1024} + self.default_gen_kwargs = {"max_new_tokens": 2048} self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs) self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs) diff --git a/mmeval/infer/ovis2.py b/mmeval/infer/ovis2.py index fa216d0c..8fb44d83 100644 --- a/mmeval/infer/ovis2.py +++ b/mmeval/infer/ovis2.py @@ -23,7 +23,7 @@ def __init__(self, args): self.dtype = getattr(args, "dtype") or torch.bfloat16 self.default_model_kwargs = {} self.default_gen_kwargs = { - "max_new_tokens": 1024, + "max_new_tokens": 2048, "do_sample": False, } self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs) diff --git a/mmeval/infer/qwen3d5.py b/mmeval/infer/qwen3d5.py index 97ca9bac..ae6510ab 100644 --- a/mmeval/infer/qwen3d5.py +++ b/mmeval/infer/qwen3d5.py @@ -22,7 +22,7 @@ def __init__(self, args): self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.dtype = getattr(args, "dtype") or "auto" self.default_model_kwargs = {"device_map": "auto"} - self.default_gen_kwargs = {"max_new_tokens": 512} + self.default_gen_kwargs = {"max_new_tokens": 2048} self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs) self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs) From 28f7071547079d1230b1196d2a72f2b7148529f5 Mon Sep 17 00:00:00 2001 From: ZTWHHH Date: Wed, 17 Jun 2026 03:19:08 +0000 Subject: [PATCH 6/6] Strip stable boundary tokens from saved responses - mmeval/infer/aria.py: strip a single trailing <|im_end|>. - mmeval/infer/glm_4d1v.py: strip a single trailing <|user|>. Useful structure such as , ..., and <|begin_of_box|>...<|end_of_box|> is preserved. mmeval/infer/qwen3d5.py does not postprocess model output. --- mmeval/infer/aria.py | 7 ++++++- mmeval/infer/glm_4d1v.py | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mmeval/infer/aria.py b/mmeval/infer/aria.py index 1b1c24e2..849f3b47 100644 --- a/mmeval/infer/aria.py +++ b/mmeval/infer/aria.py @@ -74,7 +74,12 @@ def _generate_response(self, inputs): **self.gen_kwargs, ) trimmed = output[0][inputs["input_ids"].shape[1]:] - return self.processor.decode(trimmed, skip_special_tokens=True) + response = self.processor.decode(trimmed, skip_special_tokens=True) + # Aria emits the chat-template boundary token <|im_end|> even when it + # is passed as a stop string; strip a single trailing occurrence. + if response.endswith("<|im_end|>"): + response = response[:-len("<|im_end|>")] + return response.rstrip() def run_sample(self, sample: dict): if self.args.score_target: diff --git a/mmeval/infer/glm_4d1v.py b/mmeval/infer/glm_4d1v.py index 6e3534d5..76b284de 100644 --- a/mmeval/infer/glm_4d1v.py +++ b/mmeval/infer/glm_4d1v.py @@ -65,7 +65,13 @@ def parse_input(self, message): def _generate_response(self, inputs): generated_ids = self.model.generate(**inputs, **self.gen_kwargs) trimmed = generated_ids[0][inputs["input_ids"].shape[1]:] - return self.processor.decode(trimmed, skip_special_tokens=False) + response = self.processor.decode(trimmed, skip_special_tokens=False) + # Strip the trailing chat-template boundary token. The semantic + # <|begin_of_box|>...<|end_of_box|> (and ) markers + # are kept since they delimit the model's structured answer. + if response.endswith("<|user|>"): + response = response[:-len("<|user|>")] + return response.rstrip() def run_sample(self, sample: dict): if self.args.score_target: