From 353902f2f784f2a244f513fb4060d25ea31bd1dd Mon Sep 17 00:00:00 2001 From: ZTWHHH Date: Wed, 10 Jun 2026 10:05:31 +0000 Subject: [PATCH] Add ovis2 model integration (AIDC-AI/Ovis2-8B) - mmeval/infer/ovis2.py: official-style loading and decoding - env_files/ovis2_requirements.txt: pinned reproducible env - mmeval/registry.py: series_mapping + series_infer_env_mapping entries - test_results/: passing no_media + single_image_start smoke runs HF: https://huggingface.co/AIDC-AI/Ovis2-8B GH: https://github.com/AIDC-AI/Ovis Co-Authored-By: Claude Opus 4.7 --- env_files/ovis2_requirements.txt | 12 ++ mmeval/infer/ovis2.py | 116 ++++++++++++++++++ mmeval/registry.py | 5 + .../ovis2_2026-06-10/no_media/result.json | 44 +++++++ .../single_image_start/result.json | 48 ++++++++ .../ovis2_2026-06-10/test_summary.json | 20 +++ test_results/test_ovis2.sh | 25 ++++ 7 files changed, 270 insertions(+) create mode 100644 env_files/ovis2_requirements.txt create mode 100644 mmeval/infer/ovis2.py create mode 100644 test_results/ovis2_2026-06-10/no_media/result.json create mode 100644 test_results/ovis2_2026-06-10/single_image_start/result.json create mode 100644 test_results/ovis2_2026-06-10/test_summary.json create mode 100755 test_results/test_ovis2.sh diff --git a/env_files/ovis2_requirements.txt b/env_files/ovis2_requirements.txt new file mode 100644 index 00000000..946e5b29 --- /dev/null +++ b/env_files/ovis2_requirements.txt @@ -0,0 +1,12 @@ +torch==2.4.0 +torchvision==0.19.0 +transformers==4.46.2 +accelerate +datasets +pandas +numpy==1.26.4 +Pillow +sentencepiece +protobuf +einops +timm diff --git a/mmeval/infer/ovis2.py b/mmeval/infer/ovis2.py new file mode 100644 index 00000000..fa216d0c --- /dev/null +++ b/mmeval/infer/ovis2.py @@ -0,0 +1,116 @@ +"""ovis2 — AIDC-AI/Ovis2-{1B,2B,4B,8B,16B,34B}. + +HF: https://huggingface.co/AIDC-AI/Ovis2-8B +GH: https://github.com/AIDC-AI/Ovis +Paper: https://arxiv.org/abs/2405.20797 +""" +import re +import copy + +import torch +from PIL import Image +from transformers import AutoModelForCausalLM + +from mmeval.infer.task import Task +from mmeval.utils import constants +from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs + + +class TaskRunner(Task): + def __init__(self, args): + self.args = args + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.dtype = getattr(args, "dtype") or torch.bfloat16 + self.default_model_kwargs = {} + self.default_gen_kwargs = { + "max_new_tokens": 1024, + "do_sample": False, + } + self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs) + self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs) + + super().__init__(args) + + def load_model(self, args): + self.model = AutoModelForCausalLM.from_pretrained( + args.model_name_or_path, + torch_dtype=self.dtype, + trust_remote_code=True, + multimodal_max_length=32768, + **self.model_kwargs, + ).cuda().eval() + self.text_tokenizer = self.model.get_text_tokenizer() + self.visual_tokenizer = self.model.get_visual_tokenizer() + self.max_partition = 9 + + def parse_input(self, message): + question = message["prompt"] + q_chunks = re.split(r'(<(?:image|video)>)', question) + media_list = message.get('media', []) + + text_parts = [] + images = [] + media_idx = 0 + for chunk in q_chunks: + if not chunk.strip(): + continue + if chunk == constants.image: + img = media_list[media_idx] + if isinstance(img, str): + img = Image.open(img).convert("RGB") + elif hasattr(img, "convert"): + img = img.convert("RGB") + images.append(img) + text_parts.append("") + media_idx += 1 + elif chunk == constants.video: + raise NotImplementedError("ovis2 video input not implemented") + else: + text_parts.append(chunk) + return "".join(text_parts).strip(), images + + def run_sample(self, sample: dict): + if self.args.score_target: + raise NotImplementedError( + "ovis2: score_target is not implemented yet" + ) + + ori_sample = copy.deepcopy(sample) + message = sample["messages"][0] + query, images = self.parse_input(message) + if images and "" not in query: + query = "\n" + query + + prompt, input_ids, pixel_values = self.model.preprocess_inputs( + query, images if images else None, max_partition=self.max_partition, + ) + attention_mask = torch.ne(input_ids, self.text_tokenizer.pad_token_id) + input_ids = input_ids.unsqueeze(0).to(self.model.device) + attention_mask = attention_mask.unsqueeze(0).to(self.model.device) + if pixel_values is not None: + pixel_values = [ + pixel_values.to(dtype=self.visual_tokenizer.dtype, + device=self.visual_tokenizer.device) + ] + else: + pixel_values = [None] + + with torch.inference_mode(): + output_ids = self.model.generate( + input_ids, + pixel_values=pixel_values, + attention_mask=attention_mask, + **self.gen_kwargs, + eos_token_id=self.model.generation_config.eos_token_id, + pad_token_id=self.text_tokenizer.pad_token_id, + use_cache=True, + )[0] + response = self.text_tokenizer.decode(output_ids, skip_special_tokens=True) + ori_sample["messages"].append({"role": "assistant", "response": response}) + return ori_sample + + +if __name__ == "__main__": + args = parse_args() + model_evaluator = TaskRunner(args) + model_evaluator.inference_dataset() diff --git a/mmeval/registry.py b/mmeval/registry.py index dfbf0b98..6c1d10b0 100644 --- a/mmeval/registry.py +++ b/mmeval/registry.py @@ -77,6 +77,7 @@ "doubao-seed-2-0-mini-260215", "doubao-seed-2-0-lite-260215", "doubao-seed-2-0-code-preview-260215", "doubao-seed-2-0-pro-260215"], "hunyuan_vision": ["hunyuan-vision", "hunyuan-vision-1.5-instruct", "hunyuan-t1-vision", "hunyuan-turbos-vision", "hunyuan-large-vision"], "cosmos_reason2": ["Cosmos-Reason2-2B", "Cosmos-Reason2-8B"], + "ovis2": ["Ovis2-1B", "Ovis2-2B", "Ovis2-4B", "Ovis2-8B", "Ovis2-16B", "Ovis2-34B"], } series_infer_env_mapping = { @@ -316,4 +317,8 @@ "env": os.path.join(env_dir, "cosmos_reason2"), "infer_file": "cosmos_reason2.py", }, + "ovis2": { + "env": os.path.join(env_dir, "ovis2"), + "infer_file": "ovis2.py", + }, } diff --git a/test_results/ovis2_2026-06-10/no_media/result.json b/test_results/ovis2_2026-06-10/no_media/result.json new file mode 100644 index 00000000..19b29031 --- /dev/null +++ b/test_results/ovis2_2026-06-10/no_media/result.json @@ -0,0 +1,44 @@ +[ + { + "id": 0, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the result of 1 plus 1?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the result of 1 plus 1?", + "hint": "" + }, + { + "role": "assistant", + "response": "The result of 1 plus 1 is 2." + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the capital city of China?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the capital city of China?", + "hint": "" + }, + { + "role": "assistant", + "response": "The capital city of China is Beijing." + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 1 + } +] \ No newline at end of file diff --git a/test_results/ovis2_2026-06-10/single_image_start/result.json b/test_results/ovis2_2026-06-10/single_image_start/result.json new file mode 100644 index 00000000..7e0eda71 --- /dev/null +++ b/test_results/ovis2_2026-06-10/single_image_start/result.json @@ -0,0 +1,48 @@ +[ + { + "id": 0, + "media": [ + "truck.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "The image showcases a GMC pickup truck, captured in motion on a dusty, unpaved road, suggesting off-road capabilities. The truck is painted in a sleek, metallic gray, with black accents on the wheels and front grille, enhancing its rugged appearance. The GMC logo is prominently displayed in red on the front grille, adding" + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [ + "boat.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "The image captures a sleek, white yacht gliding through the deep blue ocean. The yacht, with its polished exterior and a black stripe running along its side, is moving swiftly, creating a large wake that churns the water into white foam. The yacht is equipped with a flybridge, visible at the top, and" + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 1 + } +] \ No newline at end of file diff --git a/test_results/ovis2_2026-06-10/test_summary.json b/test_results/ovis2_2026-06-10/test_summary.json new file mode 100644 index 00000000..0085b1d8 --- /dev/null +++ b/test_results/ovis2_2026-06-10/test_summary.json @@ -0,0 +1,20 @@ +{ + "model_series": "ovis2", + "model_name": "AIDC-AI/Ovis2-8B", + "test_date": "2026-06-10", + "conda_env": "/raid/ztw/envs/ovis2", + "env_summary": "transformers==4.46.2 + flash_attn==2.6.3", + "status": "PASS", + "modalities_tested": { + "no_media": { + "status": "pass", + "rows": 2, + "result_path": "/raid/ztw/simple-mmeval-test-result/work_dirs/ovis2/2026-06-10/no_media/result.json" + }, + "single_image_start": { + "status": "pass", + "rows": 2, + "result_path": "/raid/ztw/simple-mmeval-test-result/work_dirs/ovis2/2026-06-10/single_image_start/result.json" + } + } +} \ No newline at end of file diff --git a/test_results/test_ovis2.sh b/test_results/test_ovis2.sh new file mode 100755 index 00000000..d0f22f33 --- /dev/null +++ b/test_results/test_ovis2.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Smoke test for ovis2 (model: AIDC-AI/Ovis2-8B). +set -euo pipefail + +REPO_DIR=${REPO_DIR:-/raid/ztw/simple-mmeval-skills-dev} +OUT_ROOT=${OUT_ROOT:-/raid/ztw/simple-mmeval-test-result/work_dirs/ovis2/$(date +%Y-%m-%d)} +GPU=${GPU:-0} +MODEL_ID=AIDC-AI/Ovis2-8B + +cd "$REPO_DIR" +mkdir -p "$OUT_ROOT" + +for spec in "no_media|32|" "single_image_start|64|tests/media/448"; do + IFS='|' read -r sample maxtok imgdir <<< "$spec" + out="$OUT_ROOT/$sample" + extra=() + [[ -n "$imgdir" ]] && extra=(--img_dir "$imgdir") + PYTHONPATH="$REPO_DIR" ENV_DIR=/raid/ztw/envs CUDA_VISIBLE_DEVICES="$GPU" \ + /raid/ztw/envs/ovis2/bin/python mmeval/run.py \ + --model_name_or_path "$MODEL_ID" \ + --dataset local@json --infile tests/samples/$sample.json \ + --out_dir "$out" \ + --gpu_per_parallel 1 --parallel_per_task 1 --max_new_tokens "$maxtok" \ + "${extra[@]}" +done