Add ovis2 model integration (Ovis2 family)

env_files/ovis2_requirements.txt

@@ -0,0 +1,12 @@
+torch==2.4.0
+torchvision==0.19.0
+transformers==4.46.2
+accelerate
+datasets
+pandas
+numpy==1.26.4
+Pillow
+sentencepiece
+protobuf
+einops
+timm

mmeval/infer/ovis2.py

@@ -0,0 +1,116 @@
+"""ovis2 — AIDC-AI/Ovis2-{1B,2B,4B,8B,16B,34B}.
+
+HF: https://huggingface.co/AIDC-AI/Ovis2-8B
+GH: https://github.com/AIDC-AI/Ovis
+Paper: https://arxiv.org/abs/2405.20797
+"""
+import re
+import copy
+
+import torch
+from PIL import Image
+from transformers import AutoModelForCausalLM
+
+from mmeval.infer.task import Task
+from mmeval.utils import constants
+from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs
+
+
+class TaskRunner(Task):
+    def __init__(self, args):
+        self.args = args
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.dtype = getattr(args, "dtype") or torch.bfloat16
+        self.default_model_kwargs = {}
+        self.default_gen_kwargs = {
+            "max_new_tokens": 1024,
+            "do_sample": False,
+        }
+        self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs)
+        self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs)
+
+        super().__init__(args)
+
+    def load_model(self, args):
+        self.model = AutoModelForCausalLM.from_pretrained(
+            args.model_name_or_path,
+            torch_dtype=self.dtype,
+            trust_remote_code=True,
+            multimodal_max_length=32768,
+            **self.model_kwargs,
+        ).cuda().eval()
+        self.text_tokenizer = self.model.get_text_tokenizer()
+        self.visual_tokenizer = self.model.get_visual_tokenizer()
+        self.max_partition = 9
+
+    def parse_input(self, message):
+        question = message["prompt"]
+        q_chunks = re.split(r'(<(?:image|video)>)', question)
+        media_list = message.get('media', [])
+
+        text_parts = []
+        images = []
+        media_idx = 0
+        for chunk in q_chunks:
+            if not chunk.strip():
+                continue
+            if chunk == constants.image:
+                img = media_list[media_idx]
+                if isinstance(img, str):
+                    img = Image.open(img).convert("RGB")
+                elif hasattr(img, "convert"):
+                    img = img.convert("RGB")
+                images.append(img)
+                text_parts.append("<image>")
+                media_idx += 1
+            elif chunk == constants.video:
+                raise NotImplementedError("ovis2 video input not implemented")
+            else:
+                text_parts.append(chunk)
+        return "".join(text_parts).strip(), images
+
+    def run_sample(self, sample: dict):
+        if self.args.score_target:
+            raise NotImplementedError(
+                "ovis2: score_target is not implemented yet"
+            )
+
+        ori_sample = copy.deepcopy(sample)
+        message = sample["messages"][0]
+        query, images = self.parse_input(message)
+        if images and "<image>" not in query:
+            query = "<image>\n" + query
+
+        prompt, input_ids, pixel_values = self.model.preprocess_inputs(
+            query, images if images else None, max_partition=self.max_partition,
+        )
+        attention_mask = torch.ne(input_ids, self.text_tokenizer.pad_token_id)
+        input_ids = input_ids.unsqueeze(0).to(self.model.device)
+        attention_mask = attention_mask.unsqueeze(0).to(self.model.device)
+        if pixel_values is not None:
+            pixel_values = [
+                pixel_values.to(dtype=self.visual_tokenizer.dtype,
+                                device=self.visual_tokenizer.device)
+            ]
+        else:
+            pixel_values = [None]
+
+        with torch.inference_mode():
+            output_ids = self.model.generate(
+                input_ids,
+                pixel_values=pixel_values,
+                attention_mask=attention_mask,
+                **self.gen_kwargs,
+                eos_token_id=self.model.generation_config.eos_token_id,
+                pad_token_id=self.text_tokenizer.pad_token_id,
+                use_cache=True,
+            )[0]
+        response = self.text_tokenizer.decode(output_ids, skip_special_tokens=True)
+        ori_sample["messages"].append({"role": "assistant", "response": response})
+        return ori_sample
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    model_evaluator = TaskRunner(args)
+    model_evaluator.inference_dataset()

mmeval/registry.py

@@ -77,6 +77,7 @@
                    "doubao-seed-2-0-mini-260215", "doubao-seed-2-0-lite-260215", "doubao-seed-2-0-code-preview-260215", "doubao-seed-2-0-pro-260215"],
     "hunyuan_vision": ["hunyuan-vision", "hunyuan-vision-1.5-instruct", "hunyuan-t1-vision", "hunyuan-turbos-vision", "hunyuan-large-vision"],
     "cosmos_reason2": ["Cosmos-Reason2-2B", "Cosmos-Reason2-8B"],
+    "ovis2": ["Ovis2-1B", "Ovis2-2B", "Ovis2-4B", "Ovis2-8B", "Ovis2-16B", "Ovis2-34B"],
 }
 
 series_infer_env_mapping = {
@@ -316,4 +317,8 @@
         "env": os.path.join(env_dir, "cosmos_reason2"),
         "infer_file": "cosmos_reason2.py",
     },
+    "ovis2": {
+        "env": os.path.join(env_dir, "ovis2"),
+        "infer_file": "ovis2.py",
+    },
 }

test_results/ovis2_2026-06-10/no_media/result.json

@@ -0,0 +1,44 @@
+[
+    {
+        "id": 0,
+        "media": [],
+        "messages": [
+            {
+                "role": "user",
+                "question": "What is the result of 1 plus 1?",
+                "answer": "",
+                "options": {},
+                "choices": [],
+                "prompt": "What is the result of 1 plus 1?",
+                "hint": ""
+            },
+            {
+                "role": "assistant",
+                "response": "The result of 1 plus 1 is 2."
+            }
+        ],
+        "comment": "Test 'no media' modality.",
+        "eval-id": 0
+    },
+    {
+        "id": 1,
+        "media": [],
+        "messages": [
+            {
+                "role": "user",
+                "question": "What is the capital city of China?",
+                "answer": "",
+                "options": {},
+                "choices": [],
+                "prompt": "What is the capital city of China?",
+                "hint": ""
+            },
+            {
+                "role": "assistant",
+                "response": "The capital city of China is Beijing."
+            }
+        ],
+        "comment": "Test 'no media' modality.",
+        "eval-id": 1
+    }
+]

test_results/ovis2_2026-06-10/single_image_start/result.json

@@ -0,0 +1,48 @@
+[
+    {
+        "id": 0,
+        "media": [
+            "truck.png"
+        ],
+        "messages": [
+            {
+                "role": "user",
+                "question": "<image> Please provide a detailed description of the contents shown in the image.",
+                "answer": "",
+                "options": {},
+                "choices": [],
+                "prompt": "<image> Please provide a detailed description of the contents shown in the image.",
+                "hint": ""
+            },
+            {
+                "role": "assistant",
+                "response": "The image showcases a GMC pickup truck, captured in motion on a dusty, unpaved road, suggesting off-road capabilities. The truck is painted in a sleek, metallic gray, with black accents on the wheels and front grille, enhancing its rugged appearance. The GMC logo is prominently displayed in red on the front grille, adding"
+            }
+        ],
+        "comment": "Test 'single image start' modality.",
+        "eval-id": 0
+    },
+    {
+        "id": 1,
+        "media": [
+            "boat.png"
+        ],
+        "messages": [
+            {
+                "role": "user",
+                "question": "<image> Please provide a detailed description of the contents shown in the image.",
+                "answer": "",
+                "options": {},
+                "choices": [],
+                "prompt": "<image> Please provide a detailed description of the contents shown in the image.",
+                "hint": ""
+            },
+            {
+                "role": "assistant",
+                "response": "The image captures a sleek, white yacht gliding through the deep blue ocean. The yacht, with its polished exterior and a black stripe running along its side, is moving swiftly, creating a large wake that churns the water into white foam. The yacht is equipped with a flybridge, visible at the top, and"
+            }
+        ],
+        "comment": "Test 'single image start' modality.",
+        "eval-id": 1
+    }
+]

test_results/ovis2_2026-06-10/test_summary.json

@@ -0,0 +1,20 @@
+{
+  "model_series": "ovis2",
+  "model_name": "AIDC-AI/Ovis2-8B",
+  "test_date": "2026-06-10",
+  "conda_env": "/raid/ztw/envs/ovis2",
+  "env_summary": "transformers==4.46.2 + flash_attn==2.6.3",
+  "status": "PASS",
+  "modalities_tested": {
+    "no_media": {
+      "status": "pass",
+      "rows": 2,
+      "result_path": "/raid/ztw/simple-mmeval-test-result/work_dirs/ovis2/2026-06-10/no_media/result.json"
+    },
+    "single_image_start": {
+      "status": "pass",
+      "rows": 2,
+      "result_path": "/raid/ztw/simple-mmeval-test-result/work_dirs/ovis2/2026-06-10/single_image_start/result.json"
+    }
+  }
+}

test_results/test_ovis2.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Smoke test for ovis2 (model: AIDC-AI/Ovis2-8B).
+set -euo pipefail
+
+REPO_DIR=${REPO_DIR:-/raid/ztw/simple-mmeval-skills-dev}
+OUT_ROOT=${OUT_ROOT:-/raid/ztw/simple-mmeval-test-result/work_dirs/ovis2/$(date +%Y-%m-%d)}
+GPU=${GPU:-0}
+MODEL_ID=AIDC-AI/Ovis2-8B
+
+cd "$REPO_DIR"
+mkdir -p "$OUT_ROOT"
+
+for spec in "no_media|32|" "single_image_start|64|tests/media/448"; do
+  IFS='|' read -r sample maxtok imgdir <<< "$spec"
+  out="$OUT_ROOT/$sample"
+  extra=()
+  [[ -n "$imgdir" ]] && extra=(--img_dir "$imgdir")
+  PYTHONPATH="$REPO_DIR" ENV_DIR=/raid/ztw/envs CUDA_VISIBLE_DEVICES="$GPU" \
+    /raid/ztw/envs/ovis2/bin/python mmeval/run.py \
+      --model_name_or_path "$MODEL_ID" \
+      --dataset local@json --infile tests/samples/$sample.json \
+      --out_dir "$out" \
+      --gpu_per_parallel 1 --parallel_per_task 1 --max_new_tokens "$maxtok" \
+      "${extra[@]}"
+done