diff --git a/env_files/aria_requirements.txt b/env_files/aria_requirements.txt
new file mode 100644
index 00000000..3e069fc1
--- /dev/null
+++ b/env_files/aria_requirements.txt
@@ -0,0 +1,12 @@
+torch==2.4.0
+torchvision==0.19.0
+transformers==4.49.0
+accelerate
+datasets
+pandas
+numpy==1.26.4
+Pillow
+sentencepiece
+protobuf
+einops
+timm
diff --git a/env_files/glm_4d1v_requirements.txt b/env_files/glm_4d1v_requirements.txt
new file mode 100644
index 00000000..0ef1cff7
--- /dev/null
+++ b/env_files/glm_4d1v_requirements.txt
@@ -0,0 +1,13 @@
+torch==2.6.0
+torchvision==0.21.0
+transformers==4.57.1
+accelerate
+datasets
+pandas
+numpy==1.26.4
+Pillow
+sentencepiece
+protobuf
+einops
+timm
+av
diff --git a/env_files/kimi_vl_requirements.txt b/env_files/kimi_vl_requirements.txt
new file mode 100644
index 00000000..2d02a3eb
--- /dev/null
+++ b/env_files/kimi_vl_requirements.txt
@@ -0,0 +1,14 @@
+torch==2.4.0
+torchvision==0.19.0
+transformers==4.48.2
+accelerate
+datasets
+pandas
+numpy==1.26.4
+Pillow
+sentencepiece
+protobuf
+einops
+timm
+tiktoken
+blobfile
diff --git a/env_files/minicpm_v_4d5_requirements.txt b/env_files/minicpm_v_4d5_requirements.txt
new file mode 100644
index 00000000..f2935321
--- /dev/null
+++ b/env_files/minicpm_v_4d5_requirements.txt
@@ -0,0 +1,13 @@
+torch==2.4.0
+torchvision==0.19.0
+transformers==4.55.0
+accelerate
+datasets
+pandas
+numpy==1.26.4
+Pillow
+sentencepiece
+protobuf
+einops
+timm
+decord
diff --git a/env_files/ovis2_requirements.txt b/env_files/ovis2_requirements.txt
new file mode 100644
index 00000000..946e5b29
--- /dev/null
+++ b/env_files/ovis2_requirements.txt
@@ -0,0 +1,12 @@
+torch==2.4.0
+torchvision==0.19.0
+transformers==4.46.2
+accelerate
+datasets
+pandas
+numpy==1.26.4
+Pillow
+sentencepiece
+protobuf
+einops
+timm
diff --git a/env_files/qwen3d5_requirements.txt b/env_files/qwen3d5_requirements.txt
new file mode 100644
index 00000000..72138086
--- /dev/null
+++ b/env_files/qwen3d5_requirements.txt
@@ -0,0 +1,14 @@
+torch==2.6.0
+torchvision==0.21.0
+transformers @ git+https://github.com/huggingface/transformers.git@main
+accelerate
+datasets
+pandas
+numpy==1.26.4
+Pillow
+sentencepiece
+protobuf
+einops
+timm
+qwen_vl_utils
+av
diff --git a/mmeval/infer/aria.py b/mmeval/infer/aria.py
new file mode 100644
index 00000000..849f3b47
--- /dev/null
+++ b/mmeval/infer/aria.py
@@ -0,0 +1,111 @@
+"""aria — rhymes-ai/Aria.
+
+HF: https://huggingface.co/rhymes-ai/Aria
+GH: https://github.com/rhymes-ai/Aria
+Paper: https://arxiv.org/abs/2410.05993
+"""
+import re
+import copy
+
+import torch
+from PIL import Image
+from transformers import AriaForConditionalGeneration, AriaProcessor
+
+from mmeval.infer.task import Task
+from mmeval.utils import constants
+from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs
+
+
+class TaskRunner(Task):
+ def __init__(self, args):
+ self.args = args
+ self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ self.dtype = getattr(args, "dtype") or torch.bfloat16
+ self.default_model_kwargs = {"device_map": "auto"}
+ self.default_gen_kwargs = {
+ "max_new_tokens": 2048,
+ "do_sample": True,
+ "temperature": 0.9,
+ }
+ self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs)
+ self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs)
+
+ super().__init__(args)
+
+ def load_model(self, args):
+ self.model = AriaForConditionalGeneration.from_pretrained(
+ args.model_name_or_path,
+ torch_dtype=self.dtype,
+ **self.model_kwargs,
+ ).eval()
+ self.processor = AriaProcessor.from_pretrained(args.model_name_or_path)
+
+ def parse_input(self, message):
+ question = message["prompt"]
+ q_chunks = re.split(r'(<(?:image|video)>)', question)
+ media_list = message.get('media', [])
+
+ content = []
+ images = []
+ media_idx = 0
+ for chunk in q_chunks:
+ if not chunk.strip():
+ continue
+ if chunk == constants.image:
+ img = media_list[media_idx]
+ if isinstance(img, str):
+ img = Image.open(img).convert("RGB")
+ elif hasattr(img, "convert"):
+ img = img.convert("RGB")
+ images.append(img)
+ content.append({"type": "image"})
+ media_idx += 1
+ elif chunk == constants.video:
+ raise NotImplementedError("aria video input not implemented")
+ else:
+ content.append({"type": "text", "text": chunk})
+ return [{"role": "user", "content": content}], images
+
+ def _generate_response(self, inputs):
+ output = self.model.generate(
+ **inputs,
+ stop_strings=["<|im_end|>"],
+ tokenizer=self.processor.tokenizer,
+ **self.gen_kwargs,
+ )
+ trimmed = output[0][inputs["input_ids"].shape[1]:]
+ response = self.processor.decode(trimmed, skip_special_tokens=True)
+ # Aria emits the chat-template boundary token <|im_end|> even when it
+ # is passed as a stop string; strip a single trailing occurrence.
+ if response.endswith("<|im_end|>"):
+ response = response[:-len("<|im_end|>")]
+ return response.rstrip()
+
+ def run_sample(self, sample: dict):
+ if self.args.score_target:
+ raise NotImplementedError(
+ "aria: score_target is not implemented yet"
+ )
+
+ ori_sample = copy.deepcopy(sample)
+ message = sample["messages"][0]
+ messages, images = self.parse_input(message)
+
+ text = self.processor.apply_chat_template(messages, add_generation_prompt=True)
+ proc_kwargs = {"text": text, "return_tensors": "pt"}
+ if images:
+ proc_kwargs["images"] = images
+ inputs = self.processor(**proc_kwargs)
+ if "pixel_values" in inputs:
+ inputs["pixel_values"] = inputs["pixel_values"].to(self.dtype)
+ inputs = inputs.to(self.model.device)
+
+ response = self._generate_response(inputs)
+ ori_sample["messages"].append({"role": "assistant", "response": response})
+ return ori_sample
+
+
+if __name__ == "__main__":
+ args = parse_args()
+ model_evaluator = TaskRunner(args)
+ model_evaluator.inference_dataset()
diff --git a/mmeval/infer/glm_4d1v.py b/mmeval/infer/glm_4d1v.py
new file mode 100644
index 00000000..76b284de
--- /dev/null
+++ b/mmeval/infer/glm_4d1v.py
@@ -0,0 +1,102 @@
+"""glm_4d1v — GLM-4.1V-9B-Thinking.
+
+HF: https://huggingface.co/THUDM/GLM-4.1V-9B-Thinking
+GH: https://github.com/THUDM/GLM-V
+"""
+import re
+import copy
+
+import torch
+from PIL import Image
+from transformers import AutoProcessor, Glm4vForConditionalGeneration
+
+from mmeval.infer.task import Task
+from mmeval.utils import constants
+from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs
+
+
+class TaskRunner(Task):
+ def __init__(self, args):
+ self.args = args
+ self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ self.dtype = getattr(args, "dtype") or torch.bfloat16
+ self.default_model_kwargs = {"device_map": "auto"}
+ self.default_gen_kwargs = {"max_new_tokens": 2048}
+ self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs)
+ self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs)
+
+ super().__init__(args)
+
+ def load_model(self, args):
+ self.model = Glm4vForConditionalGeneration.from_pretrained(
+ args.model_name_or_path,
+ torch_dtype=self.dtype,
+ **self.model_kwargs,
+ ).eval()
+ self.processor = AutoProcessor.from_pretrained(
+ args.model_name_or_path, use_fast=True,
+ )
+
+ def parse_input(self, message):
+ question = message["prompt"]
+ q_chunks = re.split(r'(<(?:image|video)>)', question)
+ media_list = message.get('media', [])
+
+ content = []
+ media_idx = 0
+ for chunk in q_chunks:
+ if not chunk.strip():
+ continue
+ if chunk == constants.image:
+ img = media_list[media_idx]
+ if isinstance(img, str):
+ img = Image.open(img).convert("RGB")
+ elif hasattr(img, "convert"):
+ img = img.convert("RGB")
+ content.append({"type": "image", "image": img})
+ media_idx += 1
+ elif chunk == constants.video:
+ content.append({"type": "video", "video": media_list[media_idx]})
+ media_idx += 1
+ else:
+ content.append({"type": "text", "text": chunk})
+ return [{"role": "user", "content": content}]
+
+ def _generate_response(self, inputs):
+ generated_ids = self.model.generate(**inputs, **self.gen_kwargs)
+ trimmed = generated_ids[0][inputs["input_ids"].shape[1]:]
+ response = self.processor.decode(trimmed, skip_special_tokens=False)
+ # Strip the trailing chat-template boundary token. The semantic
+ # <|begin_of_box|>...<|end_of_box|> (and ) markers
+ # are kept since they delimit the model's structured answer.
+ if response.endswith("<|user|>"):
+ response = response[:-len("<|user|>")]
+ return response.rstrip()
+
+ def run_sample(self, sample: dict):
+ if self.args.score_target:
+ raise NotImplementedError(
+ "glm_4d1v: score_target is not implemented yet"
+ )
+
+ ori_sample = copy.deepcopy(sample)
+ message = sample["messages"][0]
+ messages = self.parse_input(message)
+
+ inputs = self.processor.apply_chat_template(
+ messages,
+ tokenize=True,
+ add_generation_prompt=True,
+ return_dict=True,
+ return_tensors="pt",
+ ).to(self.model.device)
+
+ response = self._generate_response(inputs)
+ ori_sample["messages"].append({"role": "assistant", "response": response})
+ return ori_sample
+
+
+if __name__ == "__main__":
+ args = parse_args()
+ model_evaluator = TaskRunner(args)
+ model_evaluator.inference_dataset()
diff --git a/mmeval/infer/kimi_vl.py b/mmeval/infer/kimi_vl.py
new file mode 100644
index 00000000..4e80036d
--- /dev/null
+++ b/mmeval/infer/kimi_vl.py
@@ -0,0 +1,110 @@
+"""kimi_vl — Kimi-VL-A3B-{Instruct,Thinking}.
+
+HF: https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct
+GH: https://github.com/MoonshotAI/Kimi-VL
+Paper: https://arxiv.org/abs/2504.07491
+"""
+import re
+import copy
+
+import torch
+from PIL import Image
+from transformers import AutoModelForCausalLM, AutoProcessor
+
+from mmeval.infer.task import Task
+from mmeval.utils import constants
+from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs
+
+
+class TaskRunner(Task):
+ def __init__(self, args):
+ self.args = args
+ self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ self.dtype = getattr(args, "dtype") or "auto"
+ self.default_model_kwargs = {"device_map": "auto"}
+ self.default_gen_kwargs = {"max_new_tokens": 2048}
+ self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs)
+ self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs)
+
+ super().__init__(args)
+
+ def load_model(self, args):
+ self.model = AutoModelForCausalLM.from_pretrained(
+ args.model_name_or_path,
+ torch_dtype=self.dtype,
+ trust_remote_code=True,
+ **self.model_kwargs,
+ ).eval()
+ self.processor = AutoProcessor.from_pretrained(
+ args.model_name_or_path, trust_remote_code=True,
+ )
+
+ def parse_input(self, message):
+ question = message["prompt"]
+ q_chunks = re.split(r'(<(?:image|video)>)', question)
+ media_list = message.get('media', [])
+
+ content = []
+ media_idx = 0
+ for chunk in q_chunks:
+ if not chunk.strip():
+ continue
+ if chunk == constants.image:
+ content.append({"type": "image", "image": media_list[media_idx]})
+ media_idx += 1
+ elif chunk == constants.video:
+ raise NotImplementedError("kimi_vl video input not implemented")
+ else:
+ content.append({"type": "text", "text": chunk})
+ return [{"role": "user", "content": content}]
+
+ def _materialize_images(self, user_message):
+ images = []
+ for turn in user_message:
+ for item in turn["content"]:
+ if item.get("type") == "image":
+ img = item["image"]
+ if isinstance(img, str):
+ img = Image.open(img).convert("RGB")
+ elif hasattr(img, "convert"):
+ img = img.convert("RGB")
+ images.append(img)
+ return images
+
+ def _generate_response(self, inputs):
+ generated_ids = self.model.generate(**inputs, **self.gen_kwargs)
+ trimmed = [
+ out[len(inp):] for inp, out in zip(inputs.input_ids, generated_ids)
+ ]
+ return self.processor.batch_decode(
+ trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False,
+ )[0]
+
+ def run_sample(self, sample: dict):
+ if self.args.score_target:
+ raise NotImplementedError(
+ "kimi_vl: score_target is not implemented yet"
+ )
+
+ ori_sample = copy.deepcopy(sample)
+ message = sample["messages"][0]
+ user_message = self.parse_input(message)
+ images = self._materialize_images(user_message)
+
+ text = self.processor.apply_chat_template(
+ user_message, add_generation_prompt=True, return_tensors="pt",
+ )
+ proc_kwargs = {"text": text, "return_tensors": "pt", "padding": True, "truncation": True}
+ if images:
+ proc_kwargs["images"] = images
+ inputs = self.processor(**proc_kwargs).to(self.model.device)
+
+ response = self._generate_response(inputs)
+ ori_sample["messages"].append({"role": "assistant", "response": response})
+ return ori_sample
+
+
+if __name__ == "__main__":
+ args = parse_args()
+ model_evaluator = TaskRunner(args)
+ model_evaluator.inference_dataset()
diff --git a/mmeval/infer/minicpm_v_4d5.py b/mmeval/infer/minicpm_v_4d5.py
new file mode 100644
index 00000000..733abac6
--- /dev/null
+++ b/mmeval/infer/minicpm_v_4d5.py
@@ -0,0 +1,91 @@
+"""minicpm_v_4d5 — MiniCPM-V-4_5.
+
+HF: https://huggingface.co/openbmb/MiniCPM-V-4_5
+GH: https://github.com/OpenBMB/MiniCPM-o
+"""
+import re
+import copy
+
+import torch
+from PIL import Image
+from transformers import AutoModel, AutoTokenizer
+
+from mmeval.infer.task import Task
+from mmeval.utils import constants
+from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs
+
+
+class TaskRunner(Task):
+ def __init__(self, args):
+ self.args = args
+ self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ self.dtype = getattr(args, "dtype") or torch.bfloat16
+ self.default_model_kwargs = {
+ "attn_implementation": "sdpa",
+ }
+ self.default_gen_kwargs = {"max_new_tokens": 2048}
+ self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs)
+ self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs)
+
+ super().__init__(args)
+
+ def load_model(self, args):
+ self.model = AutoModel.from_pretrained(
+ args.model_name_or_path,
+ torch_dtype=self.dtype,
+ trust_remote_code=True,
+ **self.model_kwargs,
+ ).eval().cuda()
+ self.tokenizer = AutoTokenizer.from_pretrained(
+ args.model_name_or_path, trust_remote_code=True,
+ )
+
+ def parse_input(self, message):
+ question = message["prompt"]
+ q_chunks = re.split(r'(<(?:image|video)>)', question)
+ media_list = message.get('media', [])
+
+ content = []
+ media_idx = 0
+ for chunk in q_chunks:
+ if not chunk.strip():
+ continue
+ if chunk == constants.image:
+ img = media_list[media_idx]
+ if isinstance(img, str):
+ img = Image.open(img).convert("RGB")
+ elif hasattr(img, "convert"):
+ img = img.convert("RGB")
+ content.append(img)
+ media_idx += 1
+ elif chunk == constants.video:
+ raise NotImplementedError("minicpm_v_4d5 video input not implemented")
+ else:
+ content.append(chunk)
+ return [{"role": "user", "content": content}]
+
+ def run_sample(self, sample: dict):
+ if self.args.score_target:
+ raise NotImplementedError(
+ "minicpm_v_4d5: score_target is not implemented yet"
+ )
+
+ ori_sample = copy.deepcopy(sample)
+ message = sample["messages"][0]
+ msgs = self.parse_input(message)
+
+ response = self.model.chat(
+ msgs=msgs,
+ tokenizer=self.tokenizer,
+ enable_thinking=False,
+ stream=False,
+ **self.gen_kwargs,
+ )
+ ori_sample["messages"].append({"role": "assistant", "response": response})
+ return ori_sample
+
+
+if __name__ == "__main__":
+ args = parse_args()
+ model_evaluator = TaskRunner(args)
+ model_evaluator.inference_dataset()
diff --git a/mmeval/infer/ovis2.py b/mmeval/infer/ovis2.py
new file mode 100644
index 00000000..8fb44d83
--- /dev/null
+++ b/mmeval/infer/ovis2.py
@@ -0,0 +1,116 @@
+"""ovis2 — AIDC-AI/Ovis2-{1B,2B,4B,8B,16B,34B}.
+
+HF: https://huggingface.co/AIDC-AI/Ovis2-8B
+GH: https://github.com/AIDC-AI/Ovis
+Paper: https://arxiv.org/abs/2405.20797
+"""
+import re
+import copy
+
+import torch
+from PIL import Image
+from transformers import AutoModelForCausalLM
+
+from mmeval.infer.task import Task
+from mmeval.utils import constants
+from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs
+
+
+class TaskRunner(Task):
+ def __init__(self, args):
+ self.args = args
+ self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ self.dtype = getattr(args, "dtype") or torch.bfloat16
+ self.default_model_kwargs = {}
+ self.default_gen_kwargs = {
+ "max_new_tokens": 2048,
+ "do_sample": False,
+ }
+ self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs)
+ self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs)
+
+ super().__init__(args)
+
+ def load_model(self, args):
+ self.model = AutoModelForCausalLM.from_pretrained(
+ args.model_name_or_path,
+ torch_dtype=self.dtype,
+ trust_remote_code=True,
+ multimodal_max_length=32768,
+ **self.model_kwargs,
+ ).cuda().eval()
+ self.text_tokenizer = self.model.get_text_tokenizer()
+ self.visual_tokenizer = self.model.get_visual_tokenizer()
+ self.max_partition = 9
+
+ def parse_input(self, message):
+ question = message["prompt"]
+ q_chunks = re.split(r'(<(?:image|video)>)', question)
+ media_list = message.get('media', [])
+
+ text_parts = []
+ images = []
+ media_idx = 0
+ for chunk in q_chunks:
+ if not chunk.strip():
+ continue
+ if chunk == constants.image:
+ img = media_list[media_idx]
+ if isinstance(img, str):
+ img = Image.open(img).convert("RGB")
+ elif hasattr(img, "convert"):
+ img = img.convert("RGB")
+ images.append(img)
+ text_parts.append("")
+ media_idx += 1
+ elif chunk == constants.video:
+ raise NotImplementedError("ovis2 video input not implemented")
+ else:
+ text_parts.append(chunk)
+ return "".join(text_parts).strip(), images
+
+ def run_sample(self, sample: dict):
+ if self.args.score_target:
+ raise NotImplementedError(
+ "ovis2: score_target is not implemented yet"
+ )
+
+ ori_sample = copy.deepcopy(sample)
+ message = sample["messages"][0]
+ query, images = self.parse_input(message)
+ if images and "" not in query:
+ query = "\n" + query
+
+ prompt, input_ids, pixel_values = self.model.preprocess_inputs(
+ query, images if images else None, max_partition=self.max_partition,
+ )
+ attention_mask = torch.ne(input_ids, self.text_tokenizer.pad_token_id)
+ input_ids = input_ids.unsqueeze(0).to(self.model.device)
+ attention_mask = attention_mask.unsqueeze(0).to(self.model.device)
+ if pixel_values is not None:
+ pixel_values = [
+ pixel_values.to(dtype=self.visual_tokenizer.dtype,
+ device=self.visual_tokenizer.device)
+ ]
+ else:
+ pixel_values = [None]
+
+ with torch.inference_mode():
+ output_ids = self.model.generate(
+ input_ids,
+ pixel_values=pixel_values,
+ attention_mask=attention_mask,
+ **self.gen_kwargs,
+ eos_token_id=self.model.generation_config.eos_token_id,
+ pad_token_id=self.text_tokenizer.pad_token_id,
+ use_cache=True,
+ )[0]
+ response = self.text_tokenizer.decode(output_ids, skip_special_tokens=True)
+ ori_sample["messages"].append({"role": "assistant", "response": response})
+ return ori_sample
+
+
+if __name__ == "__main__":
+ args = parse_args()
+ model_evaluator = TaskRunner(args)
+ model_evaluator.inference_dataset()
diff --git a/mmeval/infer/qwen3d5.py b/mmeval/infer/qwen3d5.py
new file mode 100644
index 00000000..ae6510ab
--- /dev/null
+++ b/mmeval/infer/qwen3d5.py
@@ -0,0 +1,128 @@
+"""qwen3d5 — Qwen3.5 unified vision-language family.
+
+HF: https://huggingface.co/Qwen/Qwen3.5-9B
+GH: https://github.com/QwenLM/Qwen3.5
+Blog: https://qwen.ai/blog?id=qwen3.5
+"""
+import re
+import copy
+
+import torch
+from transformers import AutoModelForImageTextToText, AutoProcessor
+from qwen_vl_utils import process_vision_info
+
+from mmeval.infer.task import Task
+from mmeval.utils import constants
+from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs
+
+
+class TaskRunner(Task):
+ def __init__(self, args):
+ self.args = args
+ self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ self.dtype = getattr(args, "dtype") or "auto"
+ self.default_model_kwargs = {"device_map": "auto"}
+ self.default_gen_kwargs = {"max_new_tokens": 2048}
+ self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs)
+ self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs)
+
+ super().__init__(args)
+
+ def load_model(self, args):
+ self.model = AutoModelForImageTextToText.from_pretrained(
+ args.model_name_or_path,
+ dtype=self.dtype,
+ **self.model_kwargs,
+ )
+ self.processor = AutoProcessor.from_pretrained(args.model_name_or_path)
+
+ def parse_input(self, message):
+ question = message["prompt"]
+ q_chunks = re.split(r'(<(?:image|video)>)', question)
+ media_list = message.get('media', [])
+
+ messages = [{"role": "user", "content": []}]
+ media_idx = 0
+ for chunk in q_chunks:
+ if not chunk.strip():
+ continue
+ if chunk == constants.image:
+ media = media_list[media_idx]
+ media_idx += 1
+ messages[0]["content"].append({
+ "type": "image",
+ "image": media,
+ "min_pixels": 4 * 32 * 32,
+ "max_pixels": 256 * 32 * 32,
+ })
+ elif chunk == constants.video:
+ media = media_list[media_idx]
+ media_idx += 1
+ messages[0]["content"].append({
+ "type": "video",
+ "video": media,
+ "min_pixels": 4 * 32 * 32,
+ "max_pixels": 256 * 32 * 32,
+ "total_pixels": 20480 * 32 * 32,
+ })
+ else:
+ messages[0]["content"].append({"type": "text", "text": chunk})
+ return messages
+
+ def _generate_response(self, inputs):
+ generated_ids = self.model.generate(**inputs, **self.gen_kwargs)
+ generated_ids_trimmed = [
+ out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+ ]
+ return self.processor.batch_decode(
+ generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False,
+ )
+
+ def run_sample(self, sample: dict):
+ if self.args.score_target:
+ raise NotImplementedError(
+ "qwen3d5: score_target is not implemented yet"
+ )
+
+ ori_sample = copy.deepcopy(sample)
+ message = sample["messages"][0]
+
+ user_message = self.parse_input(message)
+
+ text = self.processor.apply_chat_template(
+ user_message, tokenize=False, add_generation_prompt=True
+ )
+
+ images, videos, video_kwargs = process_vision_info(
+ user_message,
+ image_patch_size=16,
+ return_video_kwargs=True,
+ return_video_metadata=True,
+ )
+
+ if videos is not None:
+ videos, video_metadatas = zip(*videos)
+ videos, video_metadatas = list(videos), list(video_metadatas)
+ else:
+ video_metadatas = None
+
+ inputs = self.processor(
+ text=text,
+ images=images,
+ videos=videos,
+ video_metadata=video_metadatas,
+ return_tensors="pt",
+ do_resize=False,
+ **video_kwargs,
+ )
+ inputs = inputs.to(self.model.device)
+
+ response = self._generate_response(inputs)
+ ori_sample["messages"].append({"role": "assistant", "response": response})
+ return ori_sample
+
+
+if __name__ == "__main__":
+ args = parse_args()
+ model_evaluator = TaskRunner(args)
+ model_evaluator.inference_dataset()
diff --git a/mmeval/registry.py b/mmeval/registry.py
index dfbf0b98..5a18dc0f 100644
--- a/mmeval/registry.py
+++ b/mmeval/registry.py
@@ -77,6 +77,12 @@
"doubao-seed-2-0-mini-260215", "doubao-seed-2-0-lite-260215", "doubao-seed-2-0-code-preview-260215", "doubao-seed-2-0-pro-260215"],
"hunyuan_vision": ["hunyuan-vision", "hunyuan-vision-1.5-instruct", "hunyuan-t1-vision", "hunyuan-turbos-vision", "hunyuan-large-vision"],
"cosmos_reason2": ["Cosmos-Reason2-2B", "Cosmos-Reason2-8B"],
+ "kimi_vl": ["Kimi-VL-A3B-Instruct", "Kimi-VL-A3B-Thinking"],
+ "minicpm_v_4d5": ["MiniCPM-V-4_5"],
+ "glm_4d1v": ["GLM-4.1V-9B-Thinking", "GLM-4.1V-9B-Base"],
+ "ovis2": ["Ovis2-1B", "Ovis2-2B", "Ovis2-4B", "Ovis2-8B"],
+ "aria": ["Aria"],
+ "qwen3d5": ["Qwen3.5-0.8B", "Qwen3.5-2B", "Qwen3.5-4B", "Qwen3.5-9B"],
}
series_infer_env_mapping = {
@@ -316,4 +322,28 @@
"env": os.path.join(env_dir, "cosmos_reason2"),
"infer_file": "cosmos_reason2.py",
},
+ "kimi_vl": {
+ "env": os.path.join(env_dir, "kimi_vl"),
+ "infer_file": "kimi_vl.py",
+ },
+ "minicpm_v_4d5": {
+ "env": os.path.join(env_dir, "minicpm_v_4d5"),
+ "infer_file": "minicpm_v_4d5.py",
+ },
+ "glm_4d1v": {
+ "env": os.path.join(env_dir, "glm_4d1v"),
+ "infer_file": "glm_4d1v.py",
+ },
+ "ovis2": {
+ "env": os.path.join(env_dir, "ovis2"),
+ "infer_file": "ovis2.py",
+ },
+ "aria": {
+ "env": os.path.join(env_dir, "aria"),
+ "infer_file": "aria.py",
+ },
+ "qwen3d5": {
+ "env": os.path.join(env_dir, "qwen3d5"),
+ "infer_file": "qwen3d5.py",
+ },
}