Add aria model integration (rhymes-ai/Aria)

env_files/aria_requirements.txt

@@ -0,0 +1,12 @@
+torch==2.4.0
+torchvision==0.19.0
+transformers==4.49.0
+accelerate
+datasets
+pandas
+numpy==1.26.4
+Pillow
+sentencepiece
+protobuf
+einops
+timm

mmeval/infer/aria.py

@@ -0,0 +1,106 @@
+"""aria — rhymes-ai/Aria.
+
+HF: https://huggingface.co/rhymes-ai/Aria
+GH: https://github.com/rhymes-ai/Aria
+Paper: https://arxiv.org/abs/2410.05993
+"""
+import re
+import copy
+
+import torch
+from PIL import Image
+from transformers import AriaForConditionalGeneration, AriaProcessor
+
+from mmeval.infer.task import Task
+from mmeval.utils import constants
+from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs
+
+
+class TaskRunner(Task):
+    def __init__(self, args):
+        self.args = args
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.dtype = getattr(args, "dtype") or torch.bfloat16
+        self.default_model_kwargs = {"device_map": "auto"}
+        self.default_gen_kwargs = {
+            "max_new_tokens": 512,
+            "do_sample": True,
+            "temperature": 0.9,
+        }
+        self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs)
+        self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs)
+
+        super().__init__(args)
+
+    def load_model(self, args):
+        self.model = AriaForConditionalGeneration.from_pretrained(
+            args.model_name_or_path,
+            torch_dtype=self.dtype,
+            **self.model_kwargs,
+        ).eval()
+        self.processor = AriaProcessor.from_pretrained(args.model_name_or_path)
+
+    def parse_input(self, message):
+        question = message["prompt"]
+        q_chunks = re.split(r'(<(?:image|video)>)', question)
+        media_list = message.get('media', [])
+
+        content = []
+        images = []
+        media_idx = 0
+        for chunk in q_chunks:
+            if not chunk.strip():
+                continue
+            if chunk == constants.image:
+                img = media_list[media_idx]
+                if isinstance(img, str):
+                    img = Image.open(img).convert("RGB")
+                elif hasattr(img, "convert"):
+                    img = img.convert("RGB")
+                images.append(img)
+                content.append({"type": "image"})
+                media_idx += 1
+            elif chunk == constants.video:
+                raise NotImplementedError("aria video input not implemented")
+            else:
+                content.append({"type": "text", "text": chunk})
+        return [{"role": "user", "content": content}], images
+
+    def _generate_response(self, inputs):
+        output = self.model.generate(
+            **inputs,
+            stop_strings=["<|im_end|>"],
+            tokenizer=self.processor.tokenizer,
+            **self.gen_kwargs,
+        )
+        trimmed = output[0][inputs["input_ids"].shape[1]:]
+        return self.processor.decode(trimmed, skip_special_tokens=True)
+
+    def run_sample(self, sample: dict):
+        if self.args.score_target:
+            raise NotImplementedError(
+                "aria: score_target is not implemented yet"
+            )
+
+        ori_sample = copy.deepcopy(sample)
+        message = sample["messages"][0]
+        messages, images = self.parse_input(message)
+
+        text = self.processor.apply_chat_template(messages, add_generation_prompt=True)
+        proc_kwargs = {"text": text, "return_tensors": "pt"}
+        if images:
+            proc_kwargs["images"] = images
+        inputs = self.processor(**proc_kwargs)
+        if "pixel_values" in inputs:
+            inputs["pixel_values"] = inputs["pixel_values"].to(self.dtype)
+        inputs = inputs.to(self.model.device)
+
+        response = self._generate_response(inputs)
+        ori_sample["messages"].append({"role": "assistant", "response": response})
+        return ori_sample
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    model_evaluator = TaskRunner(args)
+    model_evaluator.inference_dataset()

mmeval/registry.py

@@ -77,6 +77,7 @@
                    "doubao-seed-2-0-mini-260215", "doubao-seed-2-0-lite-260215", "doubao-seed-2-0-code-preview-260215", "doubao-seed-2-0-pro-260215"],
     "hunyuan_vision": ["hunyuan-vision", "hunyuan-vision-1.5-instruct", "hunyuan-t1-vision", "hunyuan-turbos-vision", "hunyuan-large-vision"],
     "cosmos_reason2": ["Cosmos-Reason2-2B", "Cosmos-Reason2-8B"],
+    "aria": ["Aria"],
 }
 
 series_infer_env_mapping = {
@@ -316,4 +317,8 @@
         "env": os.path.join(env_dir, "cosmos_reason2"),
         "infer_file": "cosmos_reason2.py",
     },
+    "aria": {
+        "env": os.path.join(env_dir, "aria"),
+        "infer_file": "aria.py",
+    },
 }

test_results/aria_2026-06-10/no_media/result.json

@@ -0,0 +1,44 @@
+[
+    {
+        "id": 0,
+        "media": [],
+        "messages": [
+            {
+                "role": "user",
+                "question": "What is the result of 1 plus 1?",
+                "answer": "",
+                "options": {},
+                "choices": [],
+                "prompt": "What is the result of 1 plus 1?",
+                "hint": ""
+            },
+            {
+                "role": "assistant",
+                "response": "The result of $1 + 1$ is $2$.<|im_end|>"
+            }
+        ],
+        "comment": "Test 'no media' modality.",
+        "eval-id": 0
+    },
+    {
+        "id": 1,
+        "media": [],
+        "messages": [
+            {
+                "role": "user",
+                "question": "What is the capital city of China?",
+                "answer": "",
+                "options": {},
+                "choices": [],
+                "prompt": "What is the capital city of China?",
+                "hint": ""
+            },
+            {
+                "role": "assistant",
+                "response": "The capital city of China is Beijing.<|im_end|>"
+            }
+        ],
+        "comment": "Test 'no media' modality.",
+        "eval-id": 1
+    }
+]

test_results/aria_2026-06-10/single_image_start/result.json

@@ -0,0 +1,48 @@
+[
+    {
+        "id": 0,
+        "media": [
+            "truck.png"
+        ],
+        "messages": [
+            {
+                "role": "user",
+                "question": "<image> Please provide a detailed description of the contents shown in the image.",
+                "answer": "",
+                "options": {},
+                "choices": [],
+                "prompt": "<image> Please provide a detailed description of the contents shown in the image.",
+                "hint": ""
+            },
+            {
+                "role": "assistant",
+                "response": "The image shows a powerful GMC pickup truck in motion, driving off-road. It's a four-door model with a sleek, modern design. The truck is a metallic shade of gray, with black rims and side mirrors. The terrain beneath it is rough, dry dirt, and it is kicking"
+            }
+        ],
+        "comment": "Test 'single image start' modality.",
+        "eval-id": 0
+    },
+    {
+        "id": 1,
+        "media": [
+            "boat.png"
+        ],
+        "messages": [
+            {
+                "role": "user",
+                "question": "<image> Please provide a detailed description of the contents shown in the image.",
+                "answer": "",
+                "options": {},
+                "choices": [],
+                "prompt": "<image> Please provide a detailed description of the contents shown in the image.",
+                "hint": ""
+            },
+            {
+                "role": "assistant",
+                "response": "The image shows a sleek, white motor yacht cruising through calm seas under a clear blue sky. The yacht features a modern design with a low profile and a streamlined hull. There is a prominent bow wave created as the boat cuts through the water, indicating it is moving at a high speed."
+            }
+        ],
+        "comment": "Test 'single image start' modality.",
+        "eval-id": 1
+    }
+]

test_results/aria_2026-06-10/test_summary.json

@@ -0,0 +1,20 @@
+{
+  "model_series": "aria",
+  "model_name": "rhymes-ai/Aria",
+  "test_date": "2026-06-10",
+  "conda_env": "/raid/ztw/envs/aria",
+  "env_summary": "transformers==4.49.0",
+  "status": "PASS",
+  "modalities_tested": {
+    "no_media": {
+      "status": "pass",
+      "rows": 2,
+      "result_path": "/raid/ztw/simple-mmeval-test-result/work_dirs/aria/2026-06-10/no_media/result.json"
+    },
+    "single_image_start": {
+      "status": "pass",
+      "rows": 2,
+      "result_path": "/raid/ztw/simple-mmeval-test-result/work_dirs/aria/2026-06-10/single_image_start/result.json"
+    }
+  }
+}

test_results/test_aria.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Smoke test for aria (model: rhymes-ai/Aria).
+set -euo pipefail
+
+REPO_DIR=${REPO_DIR:-/raid/ztw/simple-mmeval-skills-dev}
+OUT_ROOT=${OUT_ROOT:-/raid/ztw/simple-mmeval-test-result/work_dirs/aria/$(date +%Y-%m-%d)}
+GPU=${GPU:-0}
+MODEL_ID=rhymes-ai/Aria
+
+cd "$REPO_DIR"
+mkdir -p "$OUT_ROOT"
+
+for spec in "no_media|32|" "single_image_start|64|tests/media/448"; do
+  IFS='|' read -r sample maxtok imgdir <<< "$spec"
+  out="$OUT_ROOT/$sample"
+  extra=()
+  [[ -n "$imgdir" ]] && extra=(--img_dir "$imgdir")
+  PYTHONPATH="$REPO_DIR" ENV_DIR=/raid/ztw/envs CUDA_VISIBLE_DEVICES="$GPU" \
+    /raid/ztw/envs/aria/bin/python mmeval/run.py \
+      --model_name_or_path "$MODEL_ID" \
+      --dataset local@json --infile tests/samples/$sample.json \
+      --out_dir "$out" \
+      --gpu_per_parallel 1 --parallel_per_task 1 --max_new_tokens "$maxtok" \
+      "${extra[@]}"
+done