diff --git a/env_files/aria_requirements.txt b/env_files/aria_requirements.txt new file mode 100644 index 00000000..3e069fc1 --- /dev/null +++ b/env_files/aria_requirements.txt @@ -0,0 +1,12 @@ +torch==2.4.0 +torchvision==0.19.0 +transformers==4.49.0 +accelerate +datasets +pandas +numpy==1.26.4 +Pillow +sentencepiece +protobuf +einops +timm diff --git a/mmeval/infer/aria.py b/mmeval/infer/aria.py new file mode 100644 index 00000000..f3d36f89 --- /dev/null +++ b/mmeval/infer/aria.py @@ -0,0 +1,106 @@ +"""aria — rhymes-ai/Aria. + +HF: https://huggingface.co/rhymes-ai/Aria +GH: https://github.com/rhymes-ai/Aria +Paper: https://arxiv.org/abs/2410.05993 +""" +import re +import copy + +import torch +from PIL import Image +from transformers import AriaForConditionalGeneration, AriaProcessor + +from mmeval.infer.task import Task +from mmeval.utils import constants +from mmeval.utils.argparser import parse_args, parse_model_kwargs, parse_gen_kwargs + + +class TaskRunner(Task): + def __init__(self, args): + self.args = args + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.dtype = getattr(args, "dtype") or torch.bfloat16 + self.default_model_kwargs = {"device_map": "auto"} + self.default_gen_kwargs = { + "max_new_tokens": 512, + "do_sample": True, + "temperature": 0.9, + } + self.model_kwargs = parse_model_kwargs(args, self.default_model_kwargs) + self.gen_kwargs = parse_gen_kwargs(args, self.default_gen_kwargs) + + super().__init__(args) + + def load_model(self, args): + self.model = AriaForConditionalGeneration.from_pretrained( + args.model_name_or_path, + torch_dtype=self.dtype, + **self.model_kwargs, + ).eval() + self.processor = AriaProcessor.from_pretrained(args.model_name_or_path) + + def parse_input(self, message): + question = message["prompt"] + q_chunks = re.split(r'(<(?:image|video)>)', question) + media_list = message.get('media', []) + + content = [] + images = [] + media_idx = 0 + for chunk in q_chunks: + if not chunk.strip(): + continue + if chunk == constants.image: + img = media_list[media_idx] + if isinstance(img, str): + img = Image.open(img).convert("RGB") + elif hasattr(img, "convert"): + img = img.convert("RGB") + images.append(img) + content.append({"type": "image"}) + media_idx += 1 + elif chunk == constants.video: + raise NotImplementedError("aria video input not implemented") + else: + content.append({"type": "text", "text": chunk}) + return [{"role": "user", "content": content}], images + + def _generate_response(self, inputs): + output = self.model.generate( + **inputs, + stop_strings=["<|im_end|>"], + tokenizer=self.processor.tokenizer, + **self.gen_kwargs, + ) + trimmed = output[0][inputs["input_ids"].shape[1]:] + return self.processor.decode(trimmed, skip_special_tokens=True) + + def run_sample(self, sample: dict): + if self.args.score_target: + raise NotImplementedError( + "aria: score_target is not implemented yet" + ) + + ori_sample = copy.deepcopy(sample) + message = sample["messages"][0] + messages, images = self.parse_input(message) + + text = self.processor.apply_chat_template(messages, add_generation_prompt=True) + proc_kwargs = {"text": text, "return_tensors": "pt"} + if images: + proc_kwargs["images"] = images + inputs = self.processor(**proc_kwargs) + if "pixel_values" in inputs: + inputs["pixel_values"] = inputs["pixel_values"].to(self.dtype) + inputs = inputs.to(self.model.device) + + response = self._generate_response(inputs) + ori_sample["messages"].append({"role": "assistant", "response": response}) + return ori_sample + + +if __name__ == "__main__": + args = parse_args() + model_evaluator = TaskRunner(args) + model_evaluator.inference_dataset() diff --git a/mmeval/registry.py b/mmeval/registry.py index dfbf0b98..df648a5d 100644 --- a/mmeval/registry.py +++ b/mmeval/registry.py @@ -77,6 +77,7 @@ "doubao-seed-2-0-mini-260215", "doubao-seed-2-0-lite-260215", "doubao-seed-2-0-code-preview-260215", "doubao-seed-2-0-pro-260215"], "hunyuan_vision": ["hunyuan-vision", "hunyuan-vision-1.5-instruct", "hunyuan-t1-vision", "hunyuan-turbos-vision", "hunyuan-large-vision"], "cosmos_reason2": ["Cosmos-Reason2-2B", "Cosmos-Reason2-8B"], + "aria": ["Aria"], } series_infer_env_mapping = { @@ -316,4 +317,8 @@ "env": os.path.join(env_dir, "cosmos_reason2"), "infer_file": "cosmos_reason2.py", }, + "aria": { + "env": os.path.join(env_dir, "aria"), + "infer_file": "aria.py", + }, } diff --git a/test_results/aria_2026-06-10/no_media/result.json b/test_results/aria_2026-06-10/no_media/result.json new file mode 100644 index 00000000..616d6528 --- /dev/null +++ b/test_results/aria_2026-06-10/no_media/result.json @@ -0,0 +1,44 @@ +[ + { + "id": 0, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the result of 1 plus 1?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the result of 1 plus 1?", + "hint": "" + }, + { + "role": "assistant", + "response": "The result of $1 + 1$ is $2$.<|im_end|>" + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the capital city of China?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the capital city of China?", + "hint": "" + }, + { + "role": "assistant", + "response": "The capital city of China is Beijing.<|im_end|>" + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 1 + } +] \ No newline at end of file diff --git a/test_results/aria_2026-06-10/single_image_start/cache.db b/test_results/aria_2026-06-10/single_image_start/cache.db new file mode 100644 index 00000000..e69de29b diff --git a/test_results/aria_2026-06-10/single_image_start/result.json b/test_results/aria_2026-06-10/single_image_start/result.json new file mode 100644 index 00000000..55545349 --- /dev/null +++ b/test_results/aria_2026-06-10/single_image_start/result.json @@ -0,0 +1,48 @@ +[ + { + "id": 0, + "media": [ + "truck.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "The image shows a powerful GMC pickup truck in motion, driving off-road. It's a four-door model with a sleek, modern design. The truck is a metallic shade of gray, with black rims and side mirrors. The terrain beneath it is rough, dry dirt, and it is kicking" + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [ + "boat.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "The image shows a sleek, white motor yacht cruising through calm seas under a clear blue sky. The yacht features a modern design with a low profile and a streamlined hull. There is a prominent bow wave created as the boat cuts through the water, indicating it is moving at a high speed." + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 1 + } +] \ No newline at end of file diff --git a/test_results/aria_2026-06-10/test_summary.json b/test_results/aria_2026-06-10/test_summary.json new file mode 100644 index 00000000..cda1f301 --- /dev/null +++ b/test_results/aria_2026-06-10/test_summary.json @@ -0,0 +1,20 @@ +{ + "model_series": "aria", + "model_name": "rhymes-ai/Aria", + "test_date": "2026-06-10", + "conda_env": "/raid/ztw/envs/aria", + "env_summary": "transformers==4.49.0", + "status": "PASS", + "modalities_tested": { + "no_media": { + "status": "pass", + "rows": 2, + "result_path": "/raid/ztw/simple-mmeval-test-result/work_dirs/aria/2026-06-10/no_media/result.json" + }, + "single_image_start": { + "status": "pass", + "rows": 2, + "result_path": "/raid/ztw/simple-mmeval-test-result/work_dirs/aria/2026-06-10/single_image_start/result.json" + } + } +} \ No newline at end of file diff --git a/test_results/test_aria.sh b/test_results/test_aria.sh new file mode 100755 index 00000000..64c92bf0 --- /dev/null +++ b/test_results/test_aria.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Smoke test for aria (model: rhymes-ai/Aria). +set -euo pipefail + +REPO_DIR=${REPO_DIR:-/raid/ztw/simple-mmeval-skills-dev} +OUT_ROOT=${OUT_ROOT:-/raid/ztw/simple-mmeval-test-result/work_dirs/aria/$(date +%Y-%m-%d)} +GPU=${GPU:-0} +MODEL_ID=rhymes-ai/Aria + +cd "$REPO_DIR" +mkdir -p "$OUT_ROOT" + +for spec in "no_media|32|" "single_image_start|64|tests/media/448"; do + IFS='|' read -r sample maxtok imgdir <<< "$spec" + out="$OUT_ROOT/$sample" + extra=() + [[ -n "$imgdir" ]] && extra=(--img_dir "$imgdir") + PYTHONPATH="$REPO_DIR" ENV_DIR=/raid/ztw/envs CUDA_VISIBLE_DEVICES="$GPU" \ + /raid/ztw/envs/aria/bin/python mmeval/run.py \ + --model_name_or_path "$MODEL_ID" \ + --dataset local@json --infile tests/samples/$sample.json \ + --out_dir "$out" \ + --gpu_per_parallel 1 --parallel_per_task 1 --max_new_tokens "$maxtok" \ + "${extra[@]}" +done